nmxptool.c 43.7 KB
Newer Older
Matteo Quintiliani's avatar
Matteo Quintiliani committed
1 2 3 4 5 6 7 8 9
/*! \file
 *
 * \brief Nanometrics Protocol Tool
 *
 * Author:
 * 	Matteo Quintiliani
 * 	Istituto Nazionale di Geofisica e Vulcanologia - Italy
 *	quintiliani@ingv.it
 *
10
 * $Id: nmxptool.c,v 1.161 2008-03-28 20:08:26 mtheo Exp $
Matteo Quintiliani's avatar
Matteo Quintiliani committed
11
 *
Matteo Quintiliani's avatar
Matteo Quintiliani committed
12 13
 */

Matteo Quintiliani's avatar
Matteo Quintiliani committed
14 15
#include "config.h"

16 17 18
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
Matteo Quintiliani's avatar
Matteo Quintiliani committed
19
#include <errno.h>
20 21

#include <nmxp.h>
Matteo Quintiliani's avatar
Matteo Quintiliani committed
22

Matteo Quintiliani's avatar
Matteo Quintiliani committed
23
#ifndef HAVE_WINDOWS_H
Matteo Quintiliani's avatar
Matteo Quintiliani committed
24 25 26
#include <signal.h>
#endif

Matteo Quintiliani's avatar
Matteo Quintiliani committed
27 28 29 30
#ifdef HAVE_WINDOWS_H
#include <windows.h>
#endif

31
#include "nmxptool_getoptlong.h"
32 33

#ifdef HAVE_EARTHWORMOBJS
34
#include "nmxptool_ew.h"
35
#endif
36

37 38 39
#ifdef HAVE_LIBMSEED
#include <libmseed.h>
#endif
40

41 42
#ifdef HAVE___SRC_SEEDLINK_PLUGIN_H
#include "seedlink_plugin.h"
43 44
#endif

45 46
#define TIMES_FLOW_EXIT 100

47 48
#define DAP_CONDITION(params_struct) ( params_struct.start_time != 0.0 || params_struct.delay > 0 )

49 50 51
#define CURRENT_NETWORK ( (params.network)? params.network : DEFAULT_NETWORK )
#define NETCODE_OR_CURRENT_NETWORK ( (network_code[0] != 0)? network_code : CURRENT_NETWORK )

Matteo Quintiliani's avatar
Matteo Quintiliani committed
52
#define GAP_TOLLERANCE 0.001
53

54 55 56
typedef struct {
    int significant;
    double last_time;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
57
    time_t last_time_call_raw_stream;
58
    int32_t x_1;
59
    double after_start_time;
60
    NMXP_RAW_STREAM_DATA raw_stream_buffer;
61 62
} NMXPTOOL_CHAN_SEQ;

Matteo Quintiliani's avatar
Matteo Quintiliani committed
63 64

#ifndef HAVE_WINDOWS_H
65 66
static void ShutdownHandler(int sig);
static void AlarmHandler(int sig);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
67
#endif
Matteo Quintiliani's avatar
Matteo Quintiliani committed
68

69
static void save_channel_states(NMXP_CHAN_LIST_NET *chan_list, NMXPTOOL_CHAN_SEQ *chan_list_seq);
70
void load_channel_states(NMXP_CHAN_LIST_NET *chan_list, NMXPTOOL_CHAN_SEQ *chan_list_seq);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
71 72
static void flushing_raw_data_stream();

73
#ifdef HAVE_LIBMSEED
Matteo Quintiliani's avatar
Matteo Quintiliani committed
74
int nmxptool_write_miniseed(NMXP_DATA_PROCESS *pd);
75 76
int nmxptool_log_miniseed(const char *s);
int nmxptool_logerr_miniseed(const char *s);
77 78
#endif

Matteo Quintiliani's avatar
Matteo Quintiliani committed
79
#ifdef HAVE___SRC_SEEDLINK_PLUGIN_C
Matteo Quintiliani's avatar
Matteo Quintiliani committed
80
int nmxptool_send_raw_depoch(NMXP_DATA_PROCESS *pd);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
81 82
#endif

83
int nmxptool_print_seq_no(NMXP_DATA_PROCESS *pd);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
84 85

int nmxptool_check_and_log_gap(double time1, double time2, const double gap_tollerance, const char *station, const char *channel);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
86
void nmxptool_str_time_to_filename(char *str_time);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
87 88


89 90 91 92 93
/* Global variable for main program and handling terminitation program */
NMXPTOOL_PARAMS params;
int naqssock = 0;
FILE *outfile = NULL;
NMXP_CHAN_LIST *channelList = NULL;
94
NMXP_CHAN_LIST_NET *channelList_subset = NULL;
95
NMXP_CHAN_LIST_NET *channelList_subset_waste = NULL;
96
NMXPTOOL_CHAN_SEQ *channelList_Seq = NULL;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
97 98 99
int n_func_pd = 0;
int (*p_func_pd[NMXP_MAX_FUNC_PD]) (NMXP_DATA_PROCESS *);

100 101 102 103 104 105 106

#ifdef HAVE_LIBMSEED
/* Mini-SEED variables */
NMXP_DATA_SEED data_seed;
MSRecord *msr_list_chan[MAX_N_CHAN];
#endif

107
int sigcondition = 0;
108

109
int main (int argc, char **argv) {
110
    int32_t connection_time;
111
    int request_SOCKET_OK;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
112
    int i_chan, cur_chan = 0;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
113
    int to_cur_chan = 0;
114
    int request_chan;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
115
    int exitpdscondition;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
116
    int exitdapcondition;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
117
    time_t timeout_for_channel;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
118

Matteo Quintiliani's avatar
Matteo Quintiliani committed
119
    int span_interval = 10;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
120 121
    int time_to_sleep = 0;

122 123 124
    char str_start_time[200] = "";
    char str_end_time[200] = "";
    char str_pd_time[200] = "";
125 126

    NMXP_MSG_SERVER type;
127
    void *buffer = NULL;
128
    int32_t length;
129 130
    int ret;

131
    int pd_null_count = 0;
132
    int timeoutrecv_warning = 300; /* 5 minutes */
133

134 135
    int times_flow = 0;

Matteo Quintiliani's avatar
Matteo Quintiliani committed
136 137
    int recv_errno = 0;

138 139
    char filename[500] = "";
    char station_code[20] = "", channel_code[20] = "", network_code[20] = "";
140

Matteo Quintiliani's avatar
Matteo Quintiliani committed
141
    char cur_after_start_time_str[1024];
142 143
    double cur_after_start_time = DEFAULT_BUFFERED_TIME;
    int skip_current_packet = 0;
144

145
    double default_start_time = 0.0;
146
    char start_time_str[30], end_time_str[30], default_start_time_str[30];
147

148
    NMXP_DATA_PROCESS *pd = NULL;
149

Matteo Quintiliani's avatar
Matteo Quintiliani committed
150
#ifndef HAVE_WINDOWS_H
Matteo Quintiliani's avatar
Matteo Quintiliani committed
151 152 153
    /* Signal handling, use POSIX calls with standardized semantics */
    struct sigaction sa;

154
    sa.sa_handler = AlarmHandler;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
155 156 157
    sa.sa_flags = SA_RESTART;
    sigemptyset(&sa.sa_mask);
    sigaction(SIGALRM, &sa, NULL);
158

159
    sa.sa_handler = ShutdownHandler;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
160 161 162 163 164 165 166 167 168 169
    sigaction(SIGINT, &sa, NULL);
    sigaction(SIGQUIT, &sa, NULL); 
    sigaction(SIGTERM, &sa, NULL);

    sa.sa_handler = SIG_IGN;
    sigaction(SIGHUP, &sa, NULL);
    sigaction(SIGPIPE, &sa, NULL); 
#endif

    /* Default is normal output */
170
    nmxp_log(NMXP_LOG_SET, NMXP_LOG_D_NULL);
171 172 173 174 175 176

    /* Initialize params from argument values */
    if(nmxptool_getopt_long(argc, argv, &params) != 0) {
	return 1;
    }

177
    if(params.ew_configuration_file) {
Matteo Quintiliani's avatar
Matteo Quintiliani committed
178 179

#ifdef HAVE_EARTHWORMOBJS
180

181 182
	nmxp_log_init(nmxptool_ew_logit_msg, nmxptool_ew_logit_err);

Matteo Quintiliani's avatar
Matteo Quintiliani committed
183
	nmxptool_ew_configure(argv, &params);
184

Matteo Quintiliani's avatar
Matteo Quintiliani committed
185 186 187 188
	/* Check consistency of params */
	if(nmxptool_check_params(&params) != 0) {
	    return 1;
	}
189

Matteo Quintiliani's avatar
Matteo Quintiliani committed
190
#endif
191

192
    } else {
Matteo Quintiliani's avatar
Matteo Quintiliani committed
193

Matteo Quintiliani's avatar
Matteo Quintiliani committed
194 195
	nmxp_log_init(nmxp_log_stdout, nmxp_log_stderr);

196 197 198 199 200
	/* Check consistency of params */
	if(nmxptool_check_params(&params) != 0) {
	    return 1;
	}

201 202
	/* List available channels on server */
	if(params.flag_listchannels) {
203

204 205
	    nmxp_meta_chan_print(nmxp_getMetaChannelList(params.hostname, params.portnumberdap, NMXP_DATA_TIMESERIES, params.flag_request_channelinfo, params.datas_username, params.datas_password, &channelList));

206
	    return 1;
207

208 209 210 211
	} else if(params.flag_listchannelsnaqs) {

	    channelList = nmxp_getAvailableChannelList(params.hostname, params.portnumberpds, NMXP_DATA_TIMESERIES);
	    nmxp_chan_print_channelList(channelList);
212
	    return 1;
213

214 215
	}
    }
216

217
    nmxp_log(NMXP_LOG_SET, params.verbose_level);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
218 219 220
    if(params.verbose_level != DEFAULT_VERBOSE_LEVEL) {
	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_ANY, "verbose_level %d\n", params.verbose_level);
    }
221

222 223
    nmxptool_log_params(&params);

224
    NMXP_MEM_PRINT_PTR;
225

226 227
#ifdef HAVE_LIBMSEED
    if(params.flag_writeseed) {
228
	ms_loginit((void*)&nmxptool_log_miniseed, NULL, (void*)&nmxptool_logerr_miniseed, "error: ");
229 230 231 232 233 234
	/* Init mini-SEED variables */
	nmxp_data_seed_init(&data_seed);
    }
#endif


235
    /* Get list of available channels and get a subset list of params.channels */
236
    if( DAP_CONDITION(params) ) {
237
	/* From DataServer */
238 239 240 241
	if(!nmxp_getMetaChannelList(params.hostname, params.portnumberdap, NMXP_DATA_TIMESERIES,
		    params.flag_request_channelinfo, params.datas_username, params.datas_password, &channelList)) {
	    return -1;
	}
242 243 244 245 246
    } else {
	/* From NaqsServer */
	channelList = nmxp_getAvailableChannelList(params.hostname, params.portnumberpds, NMXP_DATA_TIMESERIES);
    }

247 248 249 250 251
    if(!channelList) {
	nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CHANNEL, "Channel list has not been received!\n");
	return 1;
    }

252
    channelList_subset = nmxp_chan_subset(channelList, NMXP_DATA_TIMESERIES, params.channels, CURRENT_NETWORK);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
253 254 255
    
    /* Free the complete channel list */
    if(channelList) {
256
	NMXP_MEM_FREE(channelList);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
257 258
	channelList = NULL;
    }
259 260 261

    /* Check if some channel already exists */
    if(channelList_subset->number <= 0) {
262
	nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CHANNEL, "Channels not found!\n");
263
	return 1;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
264
    } else {
265
	nmxp_chan_print_netchannelList(channelList_subset);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
266

267
	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CHANNEL, "Init channelList_Seq.\n");
268

269
	/* init channelList_Seq */
270
	channelList_Seq = (NMXPTOOL_CHAN_SEQ *) NMXP_MEM_MALLOC(sizeof(NMXPTOOL_CHAN_SEQ) * channelList_subset->number);
271
	for(i_chan = 0; i_chan < channelList_subset->number; i_chan++) {
272 273 274 275 276 277
	    channelList_Seq[i_chan].significant = 0;
	    channelList_Seq[i_chan].last_time = 0.0;
	    channelList_Seq[i_chan].last_time_call_raw_stream = 0;
	    channelList_Seq[i_chan].x_1 = 0;
	    channelList_Seq[i_chan].after_start_time = DEFAULT_BUFFERED_TIME;
	    nmxp_raw_stream_init(&(channelList_Seq[i_chan].raw_stream_buffer), params.max_tolerable_latency, params.timeoutrecv);
278 279
	}

Matteo Quintiliani's avatar
Matteo Quintiliani committed
280
#ifdef HAVE_LIBMSEED
281 282
	if(params.flag_writeseed) {
	    nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "Init mini-SEED record list.\n");
283

284 285
	    /* Init mini-SEED record list */
	    for(i_chan = 0; i_chan < channelList_subset->number; i_chan++) {
286

287 288
		nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA,
			"Init mini-SEED record for %s\n", NMXP_LOG_STR(channelList_subset->channel[i_chan].name));
289

290
		msr_list_chan[i_chan] = msr_init(NULL);
291

292 293
		/* Separate station_code and channel_code */
		if(nmxp_chan_cpy_sta_chan(channelList_subset->channel[i_chan].name, station_code, channel_code, network_code)) {
294

295 296
		    nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "%s.%s.%s\n",
			    NMXP_LOG_STR(NETCODE_OR_CURRENT_NETWORK), NMXP_LOG_STR(station_code), NMXP_LOG_STR(channel_code));
297

298 299 300
		    strcpy(msr_list_chan[i_chan]->network, NETCODE_OR_CURRENT_NETWORK);
		    strcpy(msr_list_chan[i_chan]->station, station_code);
		    strcpy(msr_list_chan[i_chan]->channel, channel_code);
301

302 303
		    msr_list_chan[i_chan]->reclen = 512;         /* byte record length */
		    msr_list_chan[i_chan]->encoding = DE_STEIM1;  /* Steim 1 compression */
304

305 306 307 308 309
		} else {
		    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CHANNEL,
			    "Channels %s error in format!\n", NMXP_LOG_STR(channelList_subset->channel[i_chan].name));
		    return 1;
		}
310

311
	    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
312 313
	}
#endif
314

315 316
    }

317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
#ifdef HAVE_EARTHWORMOBJS
    if(params.ew_configuration_file) {
	nmxptool_ew_attach();
    }
#endif

    if(params.stc == -1) {

	if(params.flag_logdata) {
	    p_func_pd[n_func_pd++] = nmxptool_print_seq_no;
	}

#ifdef HAVE_LIBMSEED
	/* Write Mini-SEED record */
	if(params.flag_writeseed) {
	    p_func_pd[n_func_pd++] = nmxptool_write_miniseed;
	}
#endif

#ifdef HAVE___SRC_SEEDLINK_PLUGIN_C
	/* Send data to SeedLink Server */
	if(params.flag_slink) {
	    p_func_pd[n_func_pd++] = nmxptool_send_raw_depoch;
	}
#endif

343 344
#ifdef HAVE_EARTHWORMOBJS
	if(params.ew_configuration_file) {
345
	    p_func_pd[n_func_pd++] = nmxptool_ew_nmx2ew;
346 347 348
	}
#endif

349 350 351
    }


352
    nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "Starting comunication.\n");
Matteo Quintiliani's avatar
Matteo Quintiliani committed
353

354 355
    times_flow = 0;

356
    while(times_flow < 2  &&  recv_errno == 0 && !sigcondition) {
357

358 359 360 361
	if(params.statefile) {
	    load_channel_states(channelList_subset, channelList_Seq);
	}

362 363 364 365 366 367 368 369 370 371 372
	if(times_flow == 0) {
	    if(params.statefile) {
		params.interval = DEFAULT_INTERVAL_INFINITE;
	    }
	} else if(times_flow == 1) {
	    params.start_time = 0.0;
	    params.end_time = 0.0;
	    params.interval = DEFAULT_INTERVAL_NO_VALUE;

	}

Matteo Quintiliani's avatar
Matteo Quintiliani committed
373
    /* Condition for starting DAP or PDS */
374 375
    if( DAP_CONDITION(params) ||
	    (times_flow == 0  &&  params.statefile && params.max_data_to_retrieve > 0 && params.interval == DEFAULT_INTERVAL_INFINITE) ) {
376 377

	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "Begin DAP Flow.\n");
378

379 380 381 382 383 384 385
	if(params.interval > 0  ||  params.interval == DEFAULT_INTERVAL_INFINITE) {
	    if(params.interval > 0) {
		params.end_time = params.start_time + params.interval;
	    } else {
		params.end_time = nmxp_data_gmtime_now();
	    }
	} else if(params.delay > 0) {
386
	    params.start_time = ((double) (time(NULL) - params.delay - span_interval) / 10.0) * 10.0;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
387 388 389
	    params.end_time = params.start_time + span_interval;
	}

390

391 392 393
	/* ************************************************************** */
	/* Start subscription protocol "DATA ACCESS PROTOCOL" version 1.0 */
	/* ************************************************************** */
394

395 396
	/* DAP Step 1: Open a socket */
	if( (naqssock = nmxp_openSocket(params.hostname, params.portnumberdap)) == NMXP_SOCKET_ERROR) {
397
	    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "Error opening socket!\n");
398 399
	    return 1;
	}
400

401 402
	/* DAP Step 2: Read connection time */
	if(nmxp_readConnectionTime(naqssock, &connection_time) != NMXP_SOCKET_OK) {
403
	    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "Error reading connection time from server!\n");
404 405
	    return 1;
	}
406

407 408
	/* DAP Step 3: Send a ConnectRequest */
	if(nmxp_sendConnectRequest(naqssock, params.datas_username, params.datas_password, connection_time) != NMXP_SOCKET_OK) {
409
	    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "Error sending connect request!\n");
410 411 412 413 414
	    return 1;
	}

	/* DAP Step 4: Wait for a Ready message */
	if(nmxp_waitReady(naqssock) != NMXP_SOCKET_OK) {
415
	    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "Error waiting Ready message!\n");
416 417
	    return 1;
	}
418

419 420
	exitdapcondition = 1;

421
	default_start_time = (params.start_time > 0.0)? params.start_time : nmxp_data_gmtime_now() - params.max_data_to_retrieve;
422

423
	while(exitdapcondition  &&  !sigcondition) {
424

425
	    /* Start loop for sending requests */
426
	    request_chan=0;
427
	    request_SOCKET_OK = NMXP_SOCKET_OK;
428

429
	    /* For each channel */
430
	    while(request_SOCKET_OK == NMXP_SOCKET_OK  &&  request_chan < channelList_subset->number  &&  exitdapcondition && !sigcondition) {
431

432
		if(params.statefile) {
433 434
		    if(channelList_Seq[request_chan].after_start_time > 0) {
			params.start_time = channelList_Seq[request_chan].after_start_time;
435
			if(params.end_time - params.start_time > params.max_data_to_retrieve) {
436
			    nmxp_data_to_str(start_time_str, params.start_time);
437
			    nmxp_data_to_str(default_start_time_str, params.end_time - params.max_data_to_retrieve);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
438
			    nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_ANY, "%s start_time changed from %s to %s\n",
439 440 441
				    NMXP_LOG_STR(channelList_subset->channel[request_chan].name),
				    NMXP_LOG_STR(start_time_str),
				    NMXP_LOG_STR(default_start_time_str));
442
			    params.start_time = params.end_time - params.max_data_to_retrieve;
443
			}
444 445 446
		    } else {
			params.start_time = default_start_time;
		    }
447 448
		    channelList_Seq[request_chan].last_time = params.start_time;
		    channelList_Seq[request_chan].significant = 1;
449

450
		}
451

452 453
		nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "nmxp_sendDataRequest %d %s (%d)\n",
			channelList_subset->channel[request_chan].key,
454 455
			NMXP_LOG_STR(channelList_subset->channel[request_chan].name),
			request_chan);
456

457 458 459
		nmxp_data_to_str(start_time_str, params.start_time);
		nmxp_data_to_str(end_time_str, params.end_time);
		nmxp_data_to_str(default_start_time_str, default_start_time);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
460
		nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "%s start_time = %s - end_time = %s - (default_start_time = %s)\n",
461 462 463 464
			NMXP_LOG_STR(channelList_subset->channel[request_chan].name),
			NMXP_LOG_STR(start_time_str),
			NMXP_LOG_STR(end_time_str),
			NMXP_LOG_STR(default_start_time_str));
465

466
		/* DAP Step 5: Send Data Request */
467
		request_SOCKET_OK = nmxp_sendDataRequest(naqssock, channelList_subset->channel[request_chan].key, (int32_t) params.start_time, (int32_t) (params.end_time + 1.0));
468

469
		if(request_SOCKET_OK == NMXP_SOCKET_OK) {
470

471 472 473 474
		    nmxp_data_to_str(str_start_time, params.start_time);
		    nmxp_data_to_str(str_end_time, params.end_time);
		    nmxptool_str_time_to_filename(str_start_time);
		    nmxptool_str_time_to_filename(str_end_time);
475

476 477
		    if(params.flag_writefile) {
			/* Open output file */
478
			if(nmxp_chan_cpy_sta_chan(channelList_subset->channel[request_chan].name, station_code, channel_code, network_code)) {
479 480 481 482 483 484 485 486
			    sprintf(filename, "%s.%s.%s_%s_%s.nmx",
				    NETCODE_OR_CURRENT_NETWORK,
				    station_code,
				    channel_code,
				    str_start_time,
				    str_end_time);
			} else {
			    sprintf(filename, "%s_%s_%s.nmx",
487
				    channelList_subset->channel[request_chan].name,
488 489 490
				    str_start_time,
				    str_end_time);
			}
491

492 493
			outfile = fopen(filename, "w");
			if(!outfile) {
494 495
			    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_EXTRA, "Can not to open file %s!",
				    NMXP_LOG_STR(filename));
496
			}
497
		    }
498 499

#ifdef HAVE_LIBMSEED
500 501
		    if(params.flag_writeseed) {
			/* Open output Mini-SEED file */
502
			if(nmxp_chan_cpy_sta_chan(channelList_subset->channel[request_chan].name, station_code, channel_code, network_code)) {
503 504 505 506 507 508 509 510
			    sprintf(data_seed.filename_mseed, "%s.%s.%s_%s_%s.miniseed",
				    NETCODE_OR_CURRENT_NETWORK,
				    station_code,
				    channel_code,
				    str_start_time,
				    str_end_time);
			} else {
			    sprintf(filename, "%s_%s_%s.miniseed",
511
				    channelList_subset->channel[request_chan].name,
512 513 514
				    str_start_time,
				    str_end_time);
			}
515

516 517
			data_seed.outfile_mseed = fopen(data_seed.filename_mseed, "w");
			if(!data_seed.outfile_mseed) {
518 519
			    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_EXTRA, "Can not to open file %s!",
				    NMXP_LOG_STR(data_seed.filename_mseed));
520
			}
521 522
		    }
#endif
523

524 525 526 527
		    if(params.flag_writefile  &&  outfile) {
			/* Compute SNCL line */

			/* Separate station_code_old_way and channel_code_old_way */
528
			if(nmxp_chan_cpy_sta_chan(channelList_subset->channel[request_chan].name, station_code, channel_code, network_code)) {
529 530 531 532 533 534 535
			    /* Write SNCL line */
			    fprintf(outfile, "%s.%s.%s.%s\n",
				    station_code,
				    NETCODE_OR_CURRENT_NETWORK,
				    channel_code,
				    (params.location)? params.location : "");
			}
536

537
		    }
538

539 540
		    /* DAP Step 6: Receive Data until receiving a Ready message */
		    ret = nmxp_receiveMessage(naqssock, &type, &buffer, &length, 0, &recv_errno);
541

542 543
		    nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "ret = %d, type = %d, length = %d, recv_errno = %d\n",
			    ret, type, length, recv_errno);
544

545
		    while(ret == NMXP_SOCKET_OK   &&    type != NMXP_MSG_READY) {
546

547 548 549
			/* Process a packet and return value in NMXP_DATA_PROCESS structure */
			pd = nmxp_processCompressedData(buffer, length, channelList_subset, NETCODE_OR_CURRENT_NETWORK);
			nmxp_data_trim(pd, params.start_time, params.end_time, 0);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
550

551 552 553
			/* To prevent to manage a packet with zero sample after nmxp_data_trim() */
			if(pd->nSamp > 0) {

554 555
			/* Log contents of last packet */
			if(params.flag_logdata) {
556
			    nmxp_data_log(pd, params.flag_logsample);
557
			}
558

559 560
			/* Set cur_chan */
			cur_chan = nmxp_chan_lookupKeyIndex(pd->key, channelList_subset);
561 562 563 564 565

			/* It is not the channel I have requested or error from nmxp_chan_lookupKeyIndex() */
			if(request_chan != cur_chan  &&  cur_chan != -1) {
			    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_ANY, "request_chan != cur_chan  %d != %d! (%d, %s) (%d, %s.%s.%s)\n",
				    request_chan, cur_chan,
566 567 568
				    channelList_subset->channel[request_chan].key,
				    NMXP_LOG_STR(channelList_subset->channel[request_chan].name),
				    pd->key, NMXP_LOG_STR(pd->network), NMXP_LOG_STR(pd->station), NMXP_LOG_STR(pd->channel));
569
			} else {
570

571
			/* Management of gaps */
572 573
			if(!channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
			    channelList_Seq[cur_chan].significant = 1;
574
			} else {
575 576 577
			    if(channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
				if(nmxptool_check_and_log_gap(pd->time, channelList_Seq[cur_chan].last_time, GAP_TOLLERANCE, pd->station, pd->channel)) {
				    channelList_Seq[cur_chan].x_1 = 0;
578
				    nmxp_data_to_str(str_pd_time, pd->time);
579 580
				    nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_EXTRA, "%s.%s x0 set to zero at %s!\n",
					    NMXP_LOG_STR(pd->station), NMXP_LOG_STR(pd->channel), NMXP_LOG_STR(str_pd_time));
581
				}
582
			    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
583
			}
584 585
			if(channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
			    channelList_Seq[cur_chan].last_time = pd->time + ((double) pd->nSamp / (double) pd->sampRate);
586
			}
587

588
#ifdef HAVE_LIBMSEED
589 590 591 592
			/* Write Mini-SEED record */
			if(params.flag_writeseed) {
			    nmxptool_write_miniseed(pd);
			}
593
#endif
594

595
#ifdef HAVE___SRC_SEEDLINK_PLUGIN_C
596 597 598 599
			/* Send data to SeedLink Server */
			if(params.flag_slink) {
			    nmxptool_send_raw_depoch(pd);
			}
600 601
#endif

602 603 604 605 606 607
#ifdef HAVE_EARTHWORMOBJS
			if(params.ew_configuration_file) {
			    nmxptool_ew_nmx2ew(pd);
			}
#endif

608 609 610 611 612 613 614 615
			if(params.flag_writefile  &&  outfile) {
			    /* Write buffer to the output file */
			    if(outfile && buffer && length > 0) {
				int32_t length_int = length;
				nmxp_data_swap_4b((int32_t *) &length_int);
				fwrite(&length_int, sizeof(length_int), 1, outfile);
				fwrite(buffer, length, 1, outfile);
			    }
616
			}
617

618 619 620
			/* Store x_1 */
			channelList_Seq[cur_chan].x_1 = pd->pDataPtr[pd->nSamp-1];

621 622
			}

623 624
			} else {
			    /* TODO: nSamp <= 0 */
625
			}
626

627

628 629
			/* Free pd->buffer */
			if(pd->buffer) {
630
			    NMXP_MEM_FREE(pd->buffer);
631 632
			    pd->buffer = NULL;
			}
633

634 635
			/* Receive Data */
			ret = nmxp_receiveMessage(naqssock, &type, &buffer, &length, 0, &recv_errno);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
636
			/* nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "ret = %d, type = %d\n", ret, type); */
637
		    }
638

639 640 641 642 643
		    if(params.flag_writefile  &&  outfile) {
			/* Close output file */
			fclose(outfile);
			outfile = NULL;
		    }
644

645
#ifdef HAVE_LIBMSEED
646 647 648 649 650
		    if(params.flag_writeseed  &&  data_seed.outfile_mseed) {
			/* Close output Mini-SEED file */
			fclose(data_seed.outfile_mseed);
			data_seed.outfile_mseed = NULL;
		    }
651
#endif
652

653 654
		} else {
		    /* TODO: error message */
655
		}
656
		request_chan++;
657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674

#ifdef HAVE_EARTHWORMOBJS
		if(params.ew_configuration_file) {

		    /* Check if we are being asked to terminate */
		    if( nmxptool_ew_check_flag_terminate() ) {
			logit ("t", "nmxptool terminating on request\n");
			nmxptool_ew_send_error(NMXPTOOL_EW_ERR_TERMREQ);
			exitdapcondition = 0;
			times_flow = TIMES_FLOW_EXIT;
		    }

		    /* Check if we need to send heartbeat message */
		    nmxptool_ew_send_heartbeat_if_needed();

		}
#endif

675
	    }
676
	    /* DAP Step 7: Repeat steps 5 and 6 for each data request */
677

678 679 680 681 682 683 684 685 686 687
	    if(params.delay > 0) {
		time_to_sleep = (params.end_time - params.start_time) - (time(NULL) - (params.start_time + params.delay + span_interval));
		if(time_to_sleep >= 0) {
		    nmxp_sleep(time_to_sleep);
		} else {
		    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "time to sleep %d sec.\n", time_to_sleep);
		    nmxp_sleep(3);
		}
		params.start_time = params.end_time;
		params.end_time = params.start_time + span_interval;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
688
	    } else {
689
		exitdapcondition = 0;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
690
	    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
691

692
	} /* END while(exitdapcondition) */
693

694 695 696 697 698 699 700 701 702 703
	/* DAP Step 8: Send a Terminate message (optional) */
	nmxp_sendTerminateSubscription(naqssock, NMXP_SHUTDOWN_NORMAL, "Bye!");

	/* DAP Step 9: Close the socket */
	nmxp_closeSocket(naqssock);

	/* ************************************************************ */
	/* End subscription protocol "DATA ACCESS PROTOCOL" version 1.0 */
	/* ************************************************************ */

704
	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "End DAP Flow.\n");
705

Matteo Quintiliani's avatar
Matteo Quintiliani committed
706
    } else {
707

708 709
	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "Begin PDS Flow.\n");

710 711 712 713 714 715 716 717 718
	/* ************************************************************* */
	/* Start subscription protocol "PRIVATE DATA STREAM" version 1.4 */
	/* ************************************************************* */

	/* PDS Step 1: Open a socket */
	naqssock = nmxp_openSocket(params.hostname, params.portnumberpds);

	if(naqssock == NMXP_SOCKET_ERROR) {
	    return 1;
719 720
	}

721 722 723 724 725 726 727 728 729 730 731
	/* PDS Step 2: Send a Connect */
	if(nmxp_sendConnect(naqssock) != NMXP_SOCKET_OK) {
	    printf("Error on sendConnect()\n");
	    return 1;
	}

	/* PDS Step 3: Receive ChannelList */
	if(nmxp_receiveChannelList(naqssock, &channelList) != NMXP_SOCKET_OK) {
	    printf("Error on receiveChannelList()\n");
	    return 1;
	}
732 733
	/* Get a subset of channel from arguments, in respect to the step 3 of PDS */
	channelList_subset_waste = nmxp_chan_subset(channelList, NMXP_DATA_TIMESERIES, params.channels, CURRENT_NETWORK);
734

Matteo Quintiliani's avatar
Matteo Quintiliani committed
735 736
	/* Free the complete channel list */
	if(channelList) {
737
	    NMXP_MEM_FREE(channelList);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
738 739 740
	    channelList = NULL;
	}

741 742
	/* TODO check if channelList_subset_waste is equal to channelList_subset and free */
	if(channelList_subset_waste) {
743
	    NMXP_MEM_FREE(channelList_subset_waste);
744 745 746
	    channelList_subset_waste = NULL;
	}
	
747
	/* PDS Step 4: Send a Request Pending (optional) */
748

749 750
	/* PDS Step 5: Send AddChannels */
	/* Request Data */
751
	nmxp_sendAddTimeSeriesChannel(naqssock, channelList_subset, params.stc, params.rate,
752
		(params.flag_buffered)? NMXP_BUFFER_YES : NMXP_BUFFER_NO, params.n_channel, params.usec, 1);
753 754 755

	/* PDS Step 6: Repeat until finished: receive and handle packets */

Matteo Quintiliani's avatar
Matteo Quintiliani committed
756 757 758 759
#ifdef HAVE_LIBMSEED
	if(params.flag_writeseed) {
	    /* Open output Mini-SEED file */
	    sprintf(data_seed.filename_mseed, "%s.realtime.miniseed",
760
		    CURRENT_NETWORK);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
761 762 763

	    data_seed.outfile_mseed = fopen(data_seed.filename_mseed, "w");
	    if(!data_seed.outfile_mseed) {
764 765
		nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_EXTRA, "Can not to open file %s!",
			NMXP_LOG_STR(data_seed.filename_mseed));
Matteo Quintiliani's avatar
Matteo Quintiliani committed
766
	    } else {
767 768
		nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "Opened file %s!\n",
			NMXP_LOG_STR(data_seed.filename_mseed));
Matteo Quintiliani's avatar
Matteo Quintiliani committed
769 770 771 772
	    }
	}
#endif

773
	/* TODO*/
Matteo Quintiliani's avatar
Matteo Quintiliani committed
774 775
	exitpdscondition = 1;

776
	skip_current_packet = 0;
777

778
	while(exitpdscondition && !sigcondition) {
Matteo Quintiliani's avatar
Matteo Quintiliani committed
779

780
	    /* Process Compressed or Decompressed Data */
Matteo Quintiliani's avatar
Matteo Quintiliani committed
781
	    pd = nmxp_receiveData(naqssock, channelList_subset, NETCODE_OR_CURRENT_NETWORK, params.timeoutrecv, &recv_errno);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
782

783 784 785 786 787 788 789 790 791 792 793
	    if(!pd) {
		pd_null_count++;
		if((pd_null_count * params.timeoutrecv) >= timeoutrecv_warning) {
		    nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_ANY, "Received %d times a null packet. (%d sec.)\n",
			    pd_null_count, pd_null_count * params.timeoutrecv);
		    pd_null_count = 0;
		}
	    } else {
		pd_null_count = 0;
	    }

794
	    if(recv_errno == 0) {
Matteo Quintiliani's avatar
Matteo Quintiliani committed
795 796
		exitpdscondition = 1;
	    } else {
797 798 799 800 801 802
#ifdef HAVE_WINDOWS_H
		if(recv_errno == WSAEWOULDBLOCK  ||  recv_errno == WSAETIMEDOUT)
#else
		if(recv_errno == EWOULDBLOCK)
#endif
		{
803 804
		    exitpdscondition = 1;
		} else {
805 806
		    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "Error receiving data. pd=%p recv_errno=%d\n",
			    pd, recv_errno);
807 808

#ifdef HAVE_EARTHWORMOBJS
809 810 811
		    if(params.ew_configuration_file) {
			nmxptool_ew_send_error(NMXPTOOL_EW_ERR_RECVDATA);
		    }
812
#endif
813 814
		    exitpdscondition = 0;
		}
Matteo Quintiliani's avatar
Matteo Quintiliani committed
815
	    }
816

817 818 819
	    if(pd) {
		/* Set cur_chan */
		cur_chan = nmxp_chan_lookupKeyIndex(pd->key, channelList_subset);
820
		if(cur_chan == -1) {
821 822
		    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_ANY, "Key %d not found in channelList_subset!\n",
			    pd->key);
823
		}
824 825
	    }

Matteo Quintiliani's avatar
Matteo Quintiliani committed
826
	    /* Log contents of last packet */
827
	    if(params.flag_logdata) {
828
		nmxp_data_log(pd, params.flag_logsample);
829
	    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
830

831
	    skip_current_packet = 0;
832
	    if(pd &&
833 834
		    (params.statefile  ||  params.buffered_time) &&
		    ( params.timeoutrecv <= 0 )
835
	      )	{
836 837
		if(params.statefile && channelList_Seq[cur_chan].after_start_time > 0.0) {
		    cur_after_start_time = channelList_Seq[cur_chan].after_start_time;
838 839 840 841 842 843
		} else if(params.buffered_time) {
		    cur_after_start_time = params.buffered_time;
		} else {
		    cur_after_start_time = DEFAULT_BUFFERED_TIME;
		}
		nmxp_data_to_str(cur_after_start_time_str, cur_after_start_time);
844 845
		nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_PACKETMAN, "cur_chan %d, cur_after_start_time %f, cur_after_start_time_str %s\n",
			cur_chan, cur_after_start_time, NMXP_LOG_STR(cur_after_start_time_str));
846 847 848
		if(pd->time + ((double) pd->nSamp / (double) pd->sampRate) >= cur_after_start_time) {
		    if(pd->time < cur_after_start_time) {
			int first_nsample_to_remove = (cur_after_start_time - pd->time) * (double) pd->sampRate;
849 850 851 852
			/* Remove the first sample in order avoiding overlap  */
			first_nsample_to_remove++;
			if(pd->nSamp > first_nsample_to_remove) {
			    pd->nSamp -= first_nsample_to_remove;
853
			    pd->time = cur_after_start_time;
854 855 856 857 858 859 860 861 862 863 864
			    pd->pDataPtr += first_nsample_to_remove;
			    pd->x0 = pd->pDataPtr[0];
			} else {
			    skip_current_packet = 1;
			}
		    }
		} else {
		    skip_current_packet = 1;
		}
	    }

Matteo Quintiliani's avatar
Matteo Quintiliani committed
865
	    if(!skip_current_packet) {
866

867 868
		/* Manage Raw Stream */
		if(params.stc == -1) {
Matteo Quintiliani's avatar
Matteo Quintiliani committed
869

870 871
		    /* cur_char is computed only for pd != NULL */
		    if(pd) {
872 873
			nmxp_raw_stream_manage(&(channelList_Seq[cur_chan].raw_stream_buffer), pd, p_func_pd, n_func_pd);
			channelList_Seq[cur_chan].last_time_call_raw_stream = nmxp_data_gmtime_now();
874
		    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
875

876 877 878 879
		    /* Check timeout for other channels */
		    if(params.timeoutrecv > 0) {
			to_cur_chan = 0;
			while(to_cur_chan < channelList_subset->number) {
880 881
			    timeout_for_channel = nmxp_data_gmtime_now() - channelList_Seq[to_cur_chan].last_time_call_raw_stream;
			    if(channelList_Seq[to_cur_chan].last_time_call_raw_stream != 0
882 883
				    && timeout_for_channel >= params.timeoutrecv) {
				nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_DOD, "Timeout for channel %s (%d sec.)\n",
884
					NMXP_LOG_STR(channelList_subset->channel[to_cur_chan].name), timeout_for_channel);
885 886
				nmxp_raw_stream_manage(&(channelList_Seq[to_cur_chan].raw_stream_buffer), NULL, p_func_pd, n_func_pd);
				channelList_Seq[to_cur_chan].last_time_call_raw_stream = nmxp_data_gmtime_now();
887 888
			    }
			    to_cur_chan++;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
889 890
			}
		    }
891 892

		} else {
893 894 895

		    if(pd) {
			/* Management of gaps */
896 897
			if(!channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
			    channelList_Seq[cur_chan].significant = 1;
898
			} else {
899 900 901
			    if(channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
				if(nmxptool_check_and_log_gap(pd->time, channelList_Seq[cur_chan].last_time, GAP_TOLLERANCE, pd->station, pd->channel)) {
				    channelList_Seq[cur_chan].x_1 = 0;
902
				    nmxp_data_to_str(str_pd_time, pd->time);
903 904
				    nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_EXTRA, "%s.%s x0 set to zero at %s!\n",
					    NMXP_LOG_STR(pd->station), NMXP_LOG_STR(pd->channel), NMXP_LOG_STR(str_pd_time));
905 906 907
				}
			    }
			}
908 909
			if(channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
			    channelList_Seq[cur_chan].last_time = pd->time + ((double) pd->nSamp / (double) pd->sampRate);
910
			}
911

Matteo Quintiliani's avatar
Matteo Quintiliani committed
912 913

#ifdef HAVE_LIBMSEED
914 915 916 917
			/* Write Mini-SEED record */
			if(params.flag_writeseed) {
			    nmxptool_write_miniseed(pd);
			}
Matteo Quintiliani's avatar
Matteo Quintiliani committed
918 919
#endif

920
#ifdef HAVE___SRC_SEEDLINK_PLUGIN_C
921 922 923 924
			/* Send data to SeedLink Server */
			if(params.flag_slink) {
			    nmxptool_send_raw_depoch(pd);
			}
925
#endif
926 927
		    }
		}
928
	    } /* End skip_current_packet condition */
929

Matteo Quintiliani's avatar
Matteo Quintiliani committed
930
	    if(pd) {
931 932
		/* Store x_1 */
		if(pd->nSamp > 0) {
933
		    channelList_Seq[cur_chan].x_1 = pd->pDataPtr[pd->nSamp-1];
934 935 936
		}
		/* Free pd->buffer */
		if(pd->buffer) {
937
		    NMXP_MEM_FREE(pd->buffer);
938 939
		    pd->buffer = NULL;
		}
Matteo Quintiliani's avatar
Matteo Quintiliani committed
940
	    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
941

Matteo Quintiliani's avatar
Matteo Quintiliani committed
942 943 944 945 946 947
#ifdef HAVE_EARTHWORMOBJS
	    if(params.ew_configuration_file) {

		/* Check if we are being asked to terminate */
		if( nmxptool_ew_check_flag_terminate() ) {
		    logit ("t", "nmxptool terminating on request\n");
948
		    nmxptool_ew_send_error(NMXPTOOL_EW_ERR_TERMREQ);