libdap Updated for version 3.20.5
libdap4 is an implementation of OPeNDAP's DAP protocol.
D4Connect.cc
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of libdap, A C++ implementation of the OPeNDAP Data
4// Access Protocol.
5
6// Copyright (c) 2002,2003 OPeNDAP, Inc.
7// Author: James Gallagher <jgallagher@opendap.org>
8// Dan Holloway <dholloway@gso.uri.edu>
9// Reza Nekovei <reza@intcomm.net>
10//
11// This library is free software; you can redistribute it and/or
12// modify it under the terms of the GNU Lesser General Public
13// License as published by the Free Software Foundation; either
14// version 2.1 of the License, or (at your option) any later version.
15//
16// This library is distributed in the hope that it will be useful,
17// but WITHOUT ANY WARRANTY; without even the implied warranty of
18// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19// Lesser General Public License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26
27// (c) COPYRIGHT URI/MIT 1994-2002
28// Please read the full copyright statement in the file COPYRIGHT_URI.
29//
30// Authors:
31// jhrg,jimg James Gallagher <jgallagher@gso.uri.edu>
32// dan Dan Holloway <dholloway@gso.uri.edu>
33// reza Reza Nekovei <reza@intcomm.net>
34
35#include "config.h"
36// #define DODS_DEBUG 1
37
38
39
40#include <cassert>
41#include <cstring>
42#include <sstream>
43
44#include "D4Connect.h"
45#include "HTTPConnect.h"
46#include "Response.h"
47#include "DMR.h"
48#include "D4Group.h"
49
50#include "D4ParserSax2.h"
51#include "chunked_stream.h"
52#include "chunked_istream.h"
53#include "D4StreamUnMarshaller.h"
54
55#include "escaping.h"
56#include "mime_util.h"
57#include "debug.h"
58
59
60
61using namespace std;
62
63namespace libdap {
64
67void D4Connect::process_dmr(DMR &dmr, Response &rs)
68{
69 DBG(cerr << "Entering D4Connect::process_dmr" << endl);
70
71 dmr.set_dap_version(rs.get_protocol());
72
73 DBG(cerr << "Entering process_data. Response.getVersion() = " << rs.get_version() << endl);
74 switch (rs.get_type()) {
75 case dap4_error: {
76#if 0
77 Error e;
78 if (!e.parse(rs.get_stream()))
79 throw InternalErr(__FILE__, __LINE__, "Could not parse the Error object returned by the server!");
80 throw e;
81#endif
82 throw InternalErr(__FILE__, __LINE__, "DAP4 errors not processed yet: FIXME!");
83 }
84
85 case web_error:
86 // Web errors (those reported in the return document's MIME header)
87 // are processed by the WWW library.
88 throw InternalErr(__FILE__, __LINE__,
89 "An error was reported by the remote httpd; this should have been processed by HTTPConnect..");
90
91 case dap4_dmr: {
92 // parse the DMR
93 try {
94 D4ParserSax2 parser;
95 // When parsing a data response, we use the permissive mode of the DMR parser
96 // (which allows Map elements to reference Arrays that are not in the DMR).
97 // Do not use that mode when parsing the DMR response - assume the DMR is
98 // valid. jhrg 4/13/16
99 parser.intern(*rs.get_cpp_stream(), &dmr);
100 }
101 catch (Error &e) {
102 cerr << "Exception: " << e.get_error_message() << endl;
103 return;
104 }
105 catch (std::exception &e) {
106 cerr << "Exception: " << e.what() << endl;
107 return;
108 }
109 catch (...) {
110 cerr << "Exception: unknown error" << endl;
111 return;
112 }
113
114 return;
115 }
116
117 default:
118 throw Error("Unknown response type");
119 }
120}
121
124void D4Connect::process_data(DMR &data, Response &rs)
125{
126 DBG(cerr << "Entering D4Connect::process_data" << endl);
127
128 assert(rs.get_cpp_stream()); // DAP4 code uses cpp streams
129
130 data.set_dap_version(rs.get_protocol());
131
132 DBG(cerr << "Entering process_data. Response.getVersion() = " << rs.get_version() << endl);
133 switch (rs.get_type()) {
134 case dap4_error: {
135#if 0
136 Error e;
137 if (!e.parse(rs.get_cpp_stream()))
138 throw InternalErr(__FILE__, __LINE__, "Could not parse the Error object returned by the server!");
139 throw e;
140#endif
141 throw InternalErr(__FILE__, __LINE__, "DAP4 errors not processed yet: FIXME!");
142 }
143
144 case web_error:
145 // Web errors (those reported in the return document's MIME header)
146 // are processed by the WWW library.
147 throw InternalErr(__FILE__, __LINE__,
148 "An error was reported by the remote httpd; this should have been processed by HTTPConnect..");
149
150 case dap4_data: {
151#if BYTE_ORDER_PREFIX
152 // Read the byte-order byte; used later on
153 char byte_order;
154 *rs.get_cpp_stream() >> byte_order;
155 //if (debug) cerr << "Byte order: " << ((byte_order) ? "big endian" : "little endian") << endl;
156#endif
157 // get a chunked input stream
158#if BYTE_ORDER_PREFIX
159 chunked_istream cis(*rs.get_cpp_stream(), 1024, byte_order);
160#else
161 chunked_istream cis(*(rs.get_cpp_stream()), CHUNK_SIZE);
162#endif
163 // parse the DMR, stopping when the boundary is found.
164 try {
165 // force chunk read
166 // get chunk size
167 int chunk_size = cis.read_next_chunk();
168 if (chunk_size < 0)
169 throw Error("Found an unexpected end of input (EOF) while reading a DAP4 data response. (1)");
170
171 // get chunk
172 char chunk[chunk_size];
173 cis.read(chunk, chunk_size);
174 // parse char * with given size
175 D4ParserSax2 parser;
176 // permissive mode allows references to Maps that are not in the response.
177 // Use this mode when parsing a data response (but not the DMR). jhrg 4/13/16
178 parser.set_strict(false);
179
180 // '-2' to discard the CRLF pair
181 parser.intern(chunk, chunk_size - 2, &data);
182 }
183 catch (Error &e) {
184 cerr << "Exception: " << e.get_error_message() << endl;
185 return;
186 }
187 catch (std::exception &e) {
188 cerr << "Exception: " << e.what() << endl;
189 return;
190 }
191 catch (...) {
192 cerr << "Exception: unknown error" << endl;
193 return;
194 }
195
196#if BYTE_ORDER_PREFIX
197 D4StreamUnMarshaller um(cis, byte_order);
198#else
199 D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
200#endif
201 data.root()->deserialize(um, data);
202
203 return;
204 }
205
206 default:
207 throw Error("Unknown response type");
208 }
209}
210
219void D4Connect::parse_mime(Response &rs)
220{
221 rs.set_version("dods/0.0"); // initial value; for backward compatibility.
222 rs.set_protocol("2.0");
223
224 istream &data_source = *rs.get_cpp_stream();
225 string mime = get_next_mime_header(data_source);
226 while (!mime.empty()) {
227 string header, value;
228 parse_mime_header(mime, header, value);
229
230 // Note that this is an ordered list
231 if (header == "content-description") {
232 DBG(cout << header << ": " << value << endl);
233 rs.set_type(get_description_type(value));
234 }
235 // Use the value of xdods-server only if no other value has been read
236 else if (header == "xdods-server" && rs.get_version() == "dods/0.0") {
237 DBG(cout << header << ": " << value << endl);
238 rs.set_version(value);
239 }
240 // This trumps 'xdods-server' and 'server'
241 else if (header == "xopendap-server") {
242 DBG(cout << header << ": " << value << endl);
243 rs.set_version(value);
244 }
245 else if (header == "xdap") {
246 DBG(cout << header << ": " << value << endl);
247 rs.set_protocol(value);
248 }
249 // Only look for 'server' if no other header supplies this info.
250 else if (rs.get_version() == "dods/0.0" && header == "server") {
251 DBG(cout << header << ": " << value << endl);
252 rs.set_version(value);
253 }
254
255 mime = get_next_mime_header(data_source);
256 }
257}
258
259// public mfuncs
260
267D4Connect::D4Connect(const string &url, string uname, string password) :
268 d_http(0), d_local(false), d_URL(""), d_UrlQueryString(""), d_server("unknown"), d_protocol("4.0")
269{
270 string name = prune_spaces(url);
271
272 // Figure out if the URL starts with 'http', if so, make sure that we
273 // talk to an instance of HTTPConnect.
274 if (name.find("http") == 0) {
275 DBG(cerr << "Connect: The identifier is an http URL" << endl);
276 d_http = new HTTPConnect(RCReader::instance());
277 d_http->set_use_cpp_streams(true);
278
279 d_URL = name;
280
281 // Find and store any CE given with the URL.
282 string::size_type dotpos = name.find('?');
283 if (dotpos != std::string::npos) { // Found a match.
284 d_URL = name.substr(0, dotpos);
285
286 d_UrlQueryString = name.substr(dotpos + 1);
287
288 if (d_UrlQueryString.find(DAP4_CE_QUERY_KEY) != std::string::npos) {
289 std::stringstream msg;
290 msg << endl;
291 msg << "WARNING: A DAP4 constraint expression key was found in the query string!" << endl;
292 msg << "The submitted dataset URL: " << name << endl;
293 msg << "Contains the query string: " << d_UrlQueryString << endl;
294 msg << "This will cause issues when making DAP4 requests that specify additional constraints. " << endl;
295 cerr << msg.str() << endl;
296 // throw Error(malformed_expr, msg.str());
297 }
298
299 }
300 }
301 else {
302 DBG(cerr << "Connect: The identifier is a local data source." << endl);
303 d_local = true; // local in this case means non-DAP
304 }
305
306 set_credentials(uname, password);
307}
308
309D4Connect::~D4Connect()
310{
311 if (d_http) delete d_http;
312}
313
314std::string D4Connect::build_dap4_ce(const string requestSuffix, const string dap4ce)
315{
316 std::stringstream url;
317 bool needsAmpersand = false;
318
319 url << d_URL << requestSuffix << "?";
320
321 if (d_UrlQueryString.length() > 0) {
322 url << d_UrlQueryString;
323 needsAmpersand = true;
324 }
325
326 if (dap4ce.length() > 0) {
327 if (needsAmpersand) url << "&";
328
329 url << DAP4_CE_QUERY_KEY << "=" << id2www_ce(dap4ce);
330 }
331
332 DBG(cerr << "D4Connect::build_dap4_ce() - Source URL: " << d_URL << endl);
333 DBG(cerr << "D4Connect::build_dap4_ce() - Source URL Query String: " << d_UrlQueryString << endl);
334 DBG(cerr << "D4Connect::build_dap4_ce() - dap4ce: " << dap4ce << endl);
335 DBG(cerr << "D4Connect::build_dap4_ce() - request URL: " << url.str() << endl);
336
337 return url.str();
338}
339
340void D4Connect::request_dmr(DMR &dmr, const string expr)
341{
342 string url = build_dap4_ce(".dmr", expr);
343
344 Response *rs = 0;
345 try {
346 rs = d_http->fetch_url(url);
347
348 d_server = rs->get_version();
349 d_protocol = rs->get_protocol();
350
351 switch (rs->get_type()) {
352 case unknown_type:
353 DBG(cerr << "Response type unknown, assuming it's a DMR response." << endl);
354 /* no break */
355 case dap4_dmr: {
356 D4ParserSax2 parser;
357 parser.intern(*rs->get_cpp_stream(), &dmr);
358 break;
359 }
360
361 case dap4_error:
362 throw InternalErr(__FILE__, __LINE__, "DAP4 errors are not processed yet.");
363
364 case web_error:
365 // We should never get here; a web error should be picked up read_url
366 // (called by fetch_url) and result in a thrown Error object.
367 throw InternalErr(__FILE__, __LINE__, "Web error found where it should never be.");
368
369 default:
370 throw InternalErr(__FILE__, __LINE__,
371 "Response type not handled (got " + long_to_string(rs->get_type()) + ").");
372 }
373 }
374 catch (...) {
375 delete rs;
376 throw;
377 }
378
379 delete rs;
380}
381
382void D4Connect::request_dap4_data(DMR &dmr, const string expr)
383{
384 string url = build_dap4_ce(".dap", expr);
385
386 Response *rs = 0;
387 try {
388 rs = d_http->fetch_url(url);
389
390 d_server = rs->get_version();
391 d_protocol = rs->get_protocol();
392
393 switch (rs->get_type()) {
394 case unknown_type:
395 DBG(cerr << "Response type unknown, assuming it's a DAP4 Data response." << endl);
396 /* no break */
397 case dap4_data: {
398#if BYTE_ORDER_PREFIX
399 istream &in = *rs->get_cpp_stream();
400 // Read the byte-order byte; used later on
401 char byte_order;
402 in >> byte_order;
403#endif
404
405 // get a chunked input stream
406#if BYTE_ORDER_PREFIX
407 chunked_istream cis(*(rs->get_cpp_stream()), 1024, byte_order);
408#else
409 chunked_istream cis(*(rs->get_cpp_stream()), CHUNK_SIZE);
410#endif
411
412 // parse the DMR, stopping when the boundary is found.
413
414 // force chunk read
415 // get chunk size
416 int chunk_size = cis.read_next_chunk();
417 if (chunk_size < 0)
418 throw Error("Found an unexpected end of input (EOF) while reading a DAP4 data response. (2)");
419
420 // get chunk
421 char chunk[chunk_size];
422 cis.read(chunk, chunk_size);
423 // parse char * with given size
424 D4ParserSax2 parser;
425 // permissive mode allows references to Maps that are not in the response.
426 parser.set_strict(false);
427 // '-2' to discard the CRLF pair
428 parser.intern(chunk, chunk_size - 2, &dmr, false /*debug*/);
429
430 // Read data and store in the DMR
431#if BYTE_ORDER_PREFIX
432 D4StreamUnMarshaller um(cis, byte_order);
433#else
434 D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
435#endif
436 dmr.root()->deserialize(um, dmr);
437
438 break;
439 }
440
441 case dap4_error:
442 throw InternalErr(__FILE__, __LINE__, "DAP4 errors are not processed yet.");
443
444 case web_error:
445 // We should never get here; a web error should be picked up read_url
446 // (called by fetch_url) and result in a thrown Error object.
447 throw InternalErr(__FILE__, __LINE__, "Web error found where it should never be.");
448
449 default:
450 throw InternalErr(__FILE__, __LINE__,
451 "Response type not handled (got " + long_to_string(rs->get_type()) + ").");
452 }
453 }
454 catch (...) {
455 delete rs;
456 throw;
457 }
458
459 delete rs;
460}
461
462void D4Connect::read_dmr(DMR &dmr, Response &rs)
463{
464 parse_mime(rs);
465 if (rs.get_type() == unknown_type) throw Error("Unknown response type.");
466
467 read_dmr_no_mime(dmr, rs);
468}
469
470void D4Connect::read_dmr_no_mime(DMR &dmr, Response &rs)
471{
472 // Assume callers know what they are doing
473 if (rs.get_type() == unknown_type) rs.set_type(dap4_dmr);
474
475 switch (rs.get_type()) {
476 case dap4_dmr:
477 process_dmr(dmr, rs);
478 d_server = rs.get_version();
479 d_protocol = dmr.dap_version();
480 break;
481 default:
482 throw Error("Expected a DAP4 DMR response.");
483 }
484}
485
486void D4Connect::read_data(DMR &data, Response &rs)
487{
488 parse_mime(rs);
489 if (rs.get_type() == unknown_type) throw Error("Unknown response type.");
490
491 read_data_no_mime(data, rs);
492}
493
494void D4Connect::read_data_no_mime(DMR &data, Response &rs)
495{
496 // Assume callers know what they are doing
497 if (rs.get_type() == unknown_type) rs.set_type(dap4_data);
498
499 switch (rs.get_type()) {
500 case dap4_data:
501 process_data(data, rs);
502 d_server = rs.get_version();
503 d_protocol = data.dap_version();
504 break;
505 default:
506 throw Error("Expected a DAP4 Data response.");
507 }
508}
509
515void D4Connect::set_credentials(string u, string p)
516{
517 if (d_http) d_http->set_credentials(u, p);
518}
519
524{
525 if (d_http) d_http->set_accept_deflate(deflate);
526}
527
533void D4Connect::set_xdap_protocol(int major, int minor)
534{
535 if (d_http) d_http->set_xdap_protocol(major, minor);
536}
537
542{
543 if (d_http) d_http->set_cache_enabled(cache);
544}
545
546bool D4Connect::is_cache_enabled()
547{
548 if (d_http)
549 return d_http->is_cache_enabled();
550 else
551 return false;
552}
553
554} // namespace libdap
void set_accept_deflate(bool deflate)
Definition D4Connect.cc:523
void set_cache_enabled(bool enabled)
Definition D4Connect.cc:541
void set_xdap_protocol(int major, int minor)
Definition D4Connect.cc:533
void set_credentials(std::string u, std::string p)
Set the credentials for responding to challenges while dereferencing URLs.
Definition D4Connect.cc:515
void set_accept_deflate(bool defalte)
HTTPResponse * fetch_url(const string &url)
void set_credentials(const string &u, const string &p)
void set_cache_enabled(bool enabled)
void set_xdap_protocol(int major, int minor)
top level DAP object to house generic methods
ObjectType get_description_type(const string &value)
Definition mime_util.cc:339
void parse_mime_header(const string &header, string &name, string &value)
Definition mime_util.cc:912
string prune_spaces(const string &name)
Definition util.cc:459
string id2www_ce(string in, const string &allowable)
Definition escaping.cc:178
string get_next_mime_header(FILE *in)
Definition mime_util.cc:838