nmxptool.c 38.8 KB
Newer Older
Matteo Quintiliani's avatar
Matteo Quintiliani committed
1
2
3
4
5
6
7
8
9
/*! \file
 *
 * \brief Nanometrics Protocol Tool
 *
 * Author:
 * 	Matteo Quintiliani
 * 	Istituto Nazionale di Geofisica e Vulcanologia - Italy
 *	quintiliani@ingv.it
 *
10
 * $Id: nmxptool.c,v 1.126 2008-02-16 15:32:23 mtheo Exp $
Matteo Quintiliani's avatar
Matteo Quintiliani committed
11
 *
Matteo Quintiliani's avatar
Matteo Quintiliani committed
12
13
 */

Matteo Quintiliani's avatar
Matteo Quintiliani committed
14
15
#include "config.h"

16
17
18
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
Matteo Quintiliani's avatar
Matteo Quintiliani committed
19
#include <errno.h>
20
21

#include <nmxp.h>
Matteo Quintiliani's avatar
Matteo Quintiliani committed
22

Matteo Quintiliani's avatar
Matteo Quintiliani committed
23
#ifndef HAVE_WINDOWS_H
Matteo Quintiliani's avatar
Matteo Quintiliani committed
24
25
26
#include <signal.h>
#endif

Matteo Quintiliani's avatar
Matteo Quintiliani committed
27
28
29
30
#ifdef HAVE_WINDOWS_H
#include <windows.h>
#endif

31
#include "nmxptool_getoptlong.h"
32
33

#ifdef HAVE_EARTHWORMOBJS
34
#include "nmxptool_ew.h"
35
#endif
36

37
38
39
#ifdef HAVE_LIBMSEED
#include <libmseed.h>
#endif
40

41
42
#ifdef HAVE___SRC_SEEDLINK_PLUGIN_H
#include "seedlink_plugin.h"
43
44
#endif

45
46
#define TIMES_FLOW_EXIT 100

47
48
#define DAP_CONDITION(params_struct) ( params_struct.start_time != 0.0 || params_struct.delay > 0 )

49
50
51
#define CURRENT_NETWORK ( (params.network)? params.network : DEFAULT_NETWORK )
#define NETCODE_OR_CURRENT_NETWORK ( (network_code[0] != 0)? network_code : CURRENT_NETWORK )

Matteo Quintiliani's avatar
Matteo Quintiliani committed
52
#define GAP_TOLLERANCE 0.001
53

54
55
56
typedef struct {
    int significant;
    double last_time;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
57
    time_t last_time_call_raw_stream;
58
    int32_t x_1;
59
    double after_start_time;
60
    NMXP_RAW_STREAM_DATA raw_stream_buffer;
61
62
} NMXPTOOL_CHAN_SEQ;

Matteo Quintiliani's avatar
Matteo Quintiliani committed
63
64

#ifndef HAVE_WINDOWS_H
Matteo Quintiliani's avatar
Matteo Quintiliani committed
65
66
static void clientShutdown(int sig);
static void clientDummyHandler(int sig);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
67
#endif
Matteo Quintiliani's avatar
Matteo Quintiliani committed
68

69
static void save_channel_states(NMXP_CHAN_LIST_NET *chan_list, NMXPTOOL_CHAN_SEQ *chan_list_seq);
70
void load_channel_states(NMXP_CHAN_LIST_NET *chan_list, NMXPTOOL_CHAN_SEQ *chan_list_seq);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
71
72
static void flushing_raw_data_stream();

73
#ifdef HAVE_LIBMSEED
Matteo Quintiliani's avatar
Matteo Quintiliani committed
74
int nmxptool_write_miniseed(NMXP_DATA_PROCESS *pd);
75
76
#endif

Matteo Quintiliani's avatar
Matteo Quintiliani committed
77
#ifdef HAVE___SRC_SEEDLINK_PLUGIN_C
Matteo Quintiliani's avatar
Matteo Quintiliani committed
78
int nmxptool_send_raw_depoch(NMXP_DATA_PROCESS *pd);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
79
80
#endif

81
int nmxptool_print_seq_no(NMXP_DATA_PROCESS *pd);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
82
83

int nmxptool_check_and_log_gap(double time1, double time2, const double gap_tollerance, const char *station, const char *channel);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
84
void nmxptool_str_time_to_filename(char *str_time);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
85
86


87
88
89
90
91
/* Global variable for main program and handling terminitation program */
NMXPTOOL_PARAMS params;
int naqssock = 0;
FILE *outfile = NULL;
NMXP_CHAN_LIST *channelList = NULL;
92
NMXP_CHAN_LIST_NET *channelList_subset = NULL;
93
NMXP_CHAN_LIST_NET *channelList_subset_waste = NULL;
94
NMXPTOOL_CHAN_SEQ *channelList_Seq = NULL;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
95
96
97
int n_func_pd = 0;
int (*p_func_pd[NMXP_MAX_FUNC_PD]) (NMXP_DATA_PROCESS *);

98
99
100
101
102
103
104
105

#ifdef HAVE_LIBMSEED
/* Mini-SEED variables */
NMXP_DATA_SEED data_seed;
MSRecord *msr_list_chan[MAX_N_CHAN];
#endif


106
int main (int argc, char **argv) {
107
    int32_t connection_time;
108
    int request_SOCKET_OK;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
109
    int i_chan, cur_chan = 0;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
110
    int to_cur_chan = 0;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
111
    int exitpdscondition;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
112
    int exitdapcondition;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
113
    time_t timeout_for_channel;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
114

Matteo Quintiliani's avatar
Matteo Quintiliani committed
115
    int span_interval = 10;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
116
117
    int time_to_sleep = 0;

118
119
    char str_start_time[200];
    char str_end_time[200];
Matteo Quintiliani's avatar
Matteo Quintiliani committed
120
    char str_pd_time[200];
121
122
123

    NMXP_MSG_SERVER type;
    void *buffer;
124
    int32_t length;
125
126
    int ret;

127
    int pd_null_count = 0;
128
    int timeoutrecv_warning = 300; /* 5 minutes */
129

Matteo Quintiliani's avatar
Matteo Quintiliani committed
130
131
    int recv_errno = 0;

132
    char filename[500];
133
    char station_code[20], channel_code[20], network_code[20];
134

Matteo Quintiliani's avatar
Matteo Quintiliani committed
135
    char cur_after_start_time_str[1024];
136
137
    double cur_after_start_time = DEFAULT_BUFFERED_TIME;
    int skip_current_packet = 0;
138

139
140
    int times_flow = 0;
    double default_start_time = 0.0;
141
    char start_time_str[30], end_time_str[30], default_start_time_str[30];
142

143
144
145
146
147
148
149
    NMXP_DATA_PROCESS *pd;

#ifdef HAVE_LIBMSEED
    /* Init mini-SEED variables */
    nmxp_data_seed_init(&data_seed);
#endif

Matteo Quintiliani's avatar
Matteo Quintiliani committed
150
#ifndef HAVE_WINDOWS_H
Matteo Quintiliani's avatar
Matteo Quintiliani committed
151
152
153
154
155
156
157
    /* Signal handling, use POSIX calls with standardized semantics */
    struct sigaction sa;

    sa.sa_handler = clientDummyHandler;
    sa.sa_flags = SA_RESTART;
    sigemptyset(&sa.sa_mask);
    sigaction(SIGALRM, &sa, NULL);
158

Matteo Quintiliani's avatar
Matteo Quintiliani committed
159
160
161
162
163
164
165
166
167
168
169
    sa.sa_handler = clientShutdown;
    sigaction(SIGINT, &sa, NULL);
    sigaction(SIGQUIT, &sa, NULL); 
    sigaction(SIGTERM, &sa, NULL);

    sa.sa_handler = SIG_IGN;
    sigaction(SIGHUP, &sa, NULL);
    sigaction(SIGPIPE, &sa, NULL); 
#endif

    /* Default is normal output */
170
    nmxp_log(NMXP_LOG_SET, NMXP_LOG_D_NULL);
171
172
173
174
175
176

    /* Initialize params from argument values */
    if(nmxptool_getopt_long(argc, argv, &params) != 0) {
	return 1;
    }

177
    if(params.ew_configuration_file) {
Matteo Quintiliani's avatar
Matteo Quintiliani committed
178
179

#ifdef HAVE_EARTHWORMOBJS
180

181
182
	nmxp_log_init(nmxptool_ew_logit_msg, nmxptool_ew_logit_err);

Matteo Quintiliani's avatar
Matteo Quintiliani committed
183
	nmxptool_ew_configure(argv, &params);
184

Matteo Quintiliani's avatar
Matteo Quintiliani committed
185
186
187
188
	/* Check consistency of params */
	if(nmxptool_check_params(&params) != 0) {
	    return 1;
	}
189

Matteo Quintiliani's avatar
Matteo Quintiliani committed
190
#endif
191

192
    } else {
Matteo Quintiliani's avatar
Matteo Quintiliani committed
193

Matteo Quintiliani's avatar
Matteo Quintiliani committed
194
195
	nmxp_log_init(nmxp_log_stdout, nmxp_log_stderr);

196
197
198
199
200
	/* Check consistency of params */
	if(nmxptool_check_params(&params) != 0) {
	    return 1;
	}

201
202
	/* List available channels on server */
	if(params.flag_listchannels) {
203

204
205
	    nmxp_meta_chan_print(nmxp_getMetaChannelList(params.hostname, params.portnumberdap, NMXP_DATA_TIMESERIES, params.flag_request_channelinfo, params.datas_username, params.datas_password, &channelList));

206
	    return 1;
207

208
209
210
211
	} else if(params.flag_listchannelsnaqs) {

	    channelList = nmxp_getAvailableChannelList(params.hostname, params.portnumberpds, NMXP_DATA_TIMESERIES);
	    nmxp_chan_print_channelList(channelList);
212
	    return 1;
213

214
215
	}
    }
216

217
218
219
    nmxp_log(NMXP_LOG_SET, params.verbose_level);
    nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_ANY, "verbose_level %d\n", params.verbose_level);

220
221
    nmxptool_log_params(&params);

222
    /* Get list of available channels and get a subset list of params.channels */
223
    if( DAP_CONDITION(params) ) {
224
225
226
227
228
229
230
	/* From DataServer */
	nmxp_getMetaChannelList(params.hostname, params.portnumberdap, NMXP_DATA_TIMESERIES, params.flag_request_channelinfo, params.datas_username, params.datas_password, &channelList);
    } else {
	/* From NaqsServer */
	channelList = nmxp_getAvailableChannelList(params.hostname, params.portnumberpds, NMXP_DATA_TIMESERIES);
    }

231
    channelList_subset = nmxp_chan_subset(channelList, NMXP_DATA_TIMESERIES, params.channels, CURRENT_NETWORK);
232
233
234

    /* Check if some channel already exists */
    if(channelList_subset->number <= 0) {
235
	nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CHANNEL, "Channels not found!\n");
236
	return 1;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
237
    } else {
238
	nmxp_chan_print_netchannelList(channelList_subset);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
239

240
	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CHANNEL, "Init channelList_Seq.\n");
241

242
243
	/* init channelList_Seq */
	channelList_Seq = (NMXPTOOL_CHAN_SEQ *) malloc(sizeof(NMXPTOOL_CHAN_SEQ) * channelList_subset->number);
244
	for(i_chan = 0; i_chan < channelList_subset->number; i_chan++) {
245
246
247
248
249
250
	    channelList_Seq[i_chan].significant = 0;
	    channelList_Seq[i_chan].last_time = 0.0;
	    channelList_Seq[i_chan].last_time_call_raw_stream = 0;
	    channelList_Seq[i_chan].x_1 = 0;
	    channelList_Seq[i_chan].after_start_time = DEFAULT_BUFFERED_TIME;
	    nmxp_raw_stream_init(&(channelList_Seq[i_chan].raw_stream_buffer), params.max_tolerable_latency, params.timeoutrecv);
251
252
	}

Matteo Quintiliani's avatar
Matteo Quintiliani committed
253
#ifdef HAVE_LIBMSEED
254
	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "Init mini-SEED record list.\n");
255
256

	/* Init mini-SEED record list */
Matteo Quintiliani's avatar
Matteo Quintiliani committed
257
	for(i_chan = 0; i_chan < channelList_subset->number; i_chan++) {
258

259
	    nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "Init mini-SEED record for %s\n", channelList_subset->channel[i_chan].name);
260
261
262
263

	    msr_list_chan[i_chan] = msr_init(NULL);

	    /* Separate station_code and channel_code */
264
	    if(nmxp_chan_cpy_sta_chan(channelList_subset->channel[i_chan].name, station_code, channel_code, network_code)) {
265

266
		nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "%s.%s.%s\n", NETCODE_OR_CURRENT_NETWORK, station_code, channel_code);
267

268
		strcpy(msr_list_chan[i_chan]->network, NETCODE_OR_CURRENT_NETWORK);
269
270
271
272
273
274
275
		strcpy(msr_list_chan[i_chan]->station, station_code);
		strcpy(msr_list_chan[i_chan]->channel, channel_code);

		msr_list_chan[i_chan]->reclen = 512;         /* byte record length */
		msr_list_chan[i_chan]->encoding = DE_STEIM1;  /* Steim 1 compression */

	    } else {
276
		nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CHANNEL, "Channels %s error in format!\n");
277
278
279
		return 1;
	    }

Matteo Quintiliani's avatar
Matteo Quintiliani committed
280
281
	}
#endif
282

283
284
285
286
287
288
289
290
    }

    /* Free the complete channel list */
    if(channelList) {
	free(channelList);
	channelList = NULL;
    }

291
292
293
294
295
296
#ifdef HAVE_EARTHWORMOBJS
	if(params.ew_configuration_file) {
	    nmxptool_ew_attach();
	}
#endif

297
    nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "Starting comunication.\n");
Matteo Quintiliani's avatar
Matteo Quintiliani committed
298

299
300
301
302
    times_flow = 0;

    while(times_flow < 2) {

303
304
305
306
	if(params.statefile) {
	    load_channel_states(channelList_subset, channelList_Seq);
	}

307
308
309
310
311
312
313
314
315
316
317
	if(times_flow == 0) {
	    if(params.statefile) {
		params.interval = DEFAULT_INTERVAL_INFINITE;
	    }
	} else if(times_flow == 1) {
	    params.start_time = 0.0;
	    params.end_time = 0.0;
	    params.interval = DEFAULT_INTERVAL_NO_VALUE;

	}

318
    /* TODO condition starting DAP or PDS */
319
320
321
    if( DAP_CONDITION(params) || (times_flow == 0  &&  params.statefile  && params.interval == DEFAULT_INTERVAL_INFINITE) ) {

	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "Begin DAP Flow.\n");
322

323
324
325
326
327
328
329
	if(params.interval > 0  ||  params.interval == DEFAULT_INTERVAL_INFINITE) {
	    if(params.interval > 0) {
		params.end_time = params.start_time + params.interval;
	    } else {
		params.end_time = nmxp_data_gmtime_now();
	    }
	} else if(params.delay > 0) {
330
	    params.start_time = ((double) (time(NULL) - params.delay - span_interval) / 10.0) * 10.0;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
331
332
333
	    params.end_time = params.start_time + span_interval;
	}

334

335
336
337
	/* ************************************************************** */
	/* Start subscription protocol "DATA ACCESS PROTOCOL" version 1.0 */
	/* ************************************************************** */
338

339
340
	/* DAP Step 1: Open a socket */
	if( (naqssock = nmxp_openSocket(params.hostname, params.portnumberdap)) == NMXP_SOCKET_ERROR) {
341
	    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "Error opening socket!\n");
342
343
	    return 1;
	}
344

345
346
	/* DAP Step 2: Read connection time */
	if(nmxp_readConnectionTime(naqssock, &connection_time) != NMXP_SOCKET_OK) {
347
	    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "Error reading connection time from server!\n");
348
349
	    return 1;
	}
350

351
352
	/* DAP Step 3: Send a ConnectRequest */
	if(nmxp_sendConnectRequest(naqssock, params.datas_username, params.datas_password, connection_time) != NMXP_SOCKET_OK) {
353
	    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "Error sending connect request!\n");
354
355
356
357
358
	    return 1;
	}

	/* DAP Step 4: Wait for a Ready message */
	if(nmxp_waitReady(naqssock) != NMXP_SOCKET_OK) {
359
	    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "Error waiting Ready message!\n");
360
361
	    return 1;
	}
362

363
364
	exitdapcondition = 1;

365
	default_start_time = (params.start_time > 0.0)? params.start_time : nmxp_data_gmtime_now() - params.max_data_to_retrieve;
366

367
	while(exitdapcondition) {
368

369
370
371
	    /* Start loop for sending requests */
	    i_chan=0;
	    request_SOCKET_OK = NMXP_SOCKET_OK;
372

373
	    while(request_SOCKET_OK == NMXP_SOCKET_OK  &&  i_chan < channelList_subset->number) {
374

375
		if(params.statefile) {
376
377
		    if(channelList_Seq[i_chan].after_start_time > 0) {
			params.start_time = channelList_Seq[i_chan].after_start_time;
378
			if(params.end_time - params.start_time > params.max_data_to_retrieve) {
379
			    nmxp_data_to_str(start_time_str, params.start_time);
380
			    nmxp_data_to_str(default_start_time_str, params.end_time - params.max_data_to_retrieve);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
381
382
			    nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_ANY, "%s start_time changed from %s to %s\n",
				    channelList_subset->channel[i_chan].name, start_time_str, default_start_time_str);
383
			    params.start_time = params.end_time - params.max_data_to_retrieve;
384
			}
385
386
387
		    } else {
			params.start_time = default_start_time;
		    }
388
389
		    channelList_Seq[i_chan].last_time = params.start_time;
		    channelList_Seq[i_chan].significant = 1;
390

391
		}
392

393
394
395
		nmxp_data_to_str(start_time_str, params.start_time);
		nmxp_data_to_str(end_time_str, params.end_time);
		nmxp_data_to_str(default_start_time_str, default_start_time);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
396
397
		nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "%s start_time = %s - end_time = %s - (default_start_time = %s)\n",
			channelList_subset->channel[i_chan].name, start_time_str, end_time_str, default_start_time_str);
398

399
400
		/* DAP Step 5: Send Data Request */
		request_SOCKET_OK = nmxp_sendDataRequest(naqssock, channelList_subset->channel[i_chan].key, (int32_t) params.start_time, (int32_t) (params.end_time + 1.0));
401

402
		if(request_SOCKET_OK == NMXP_SOCKET_OK) {
403

404
405
406
407
		    nmxp_data_to_str(str_start_time, params.start_time);
		    nmxp_data_to_str(str_end_time, params.end_time);
		    nmxptool_str_time_to_filename(str_start_time);
		    nmxptool_str_time_to_filename(str_end_time);
408

409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
		    if(params.flag_writefile) {
			/* Open output file */
			if(nmxp_chan_cpy_sta_chan(channelList_subset->channel[i_chan].name, station_code, channel_code, network_code)) {
			    sprintf(filename, "%s.%s.%s_%s_%s.nmx",
				    NETCODE_OR_CURRENT_NETWORK,
				    station_code,
				    channel_code,
				    str_start_time,
				    str_end_time);
			} else {
			    sprintf(filename, "%s_%s_%s.nmx",
				    channelList_subset->channel[i_chan].name,
				    str_start_time,
				    str_end_time);
			}
424

425
426
427
428
			outfile = fopen(filename, "w");
			if(!outfile) {
			    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_EXTRA, "Can not to open file %s!", filename);
			}
429
		    }
430
431

#ifdef HAVE_LIBMSEED
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
		    if(params.flag_writeseed) {
			/* Open output Mini-SEED file */
			if(nmxp_chan_cpy_sta_chan(channelList_subset->channel[i_chan].name, station_code, channel_code, network_code)) {
			    sprintf(data_seed.filename_mseed, "%s.%s.%s_%s_%s.miniseed",
				    NETCODE_OR_CURRENT_NETWORK,
				    station_code,
				    channel_code,
				    str_start_time,
				    str_end_time);
			} else {
			    sprintf(filename, "%s_%s_%s.miniseed",
				    channelList_subset->channel[i_chan].name,
				    str_start_time,
				    str_end_time);
			}
447

448
449
450
451
			data_seed.outfile_mseed = fopen(data_seed.filename_mseed, "w");
			if(!data_seed.outfile_mseed) {
			    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_EXTRA, "Can not to open file %s!", data_seed.filename_mseed);
			}
452
453
		    }
#endif
454

455
456
457
458
459
460
461
462
463
464
465
466
		    if(params.flag_writefile  &&  outfile) {
			/* Compute SNCL line */

			/* Separate station_code_old_way and channel_code_old_way */
			if(nmxp_chan_cpy_sta_chan(channelList_subset->channel[i_chan].name, station_code, channel_code, network_code)) {
			    /* Write SNCL line */
			    fprintf(outfile, "%s.%s.%s.%s\n",
				    station_code,
				    NETCODE_OR_CURRENT_NETWORK,
				    channel_code,
				    (params.location)? params.location : "");
			}
467

468
		    }
469

470
471
		    /* DAP Step 6: Receive Data until receiving a Ready message */
		    ret = nmxp_receiveMessage(naqssock, &type, &buffer, &length, 0, &recv_errno);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
472
		    /* nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "ret = %d, type = %d\n", ret, type); */
473

474
		    while(ret == NMXP_SOCKET_OK   &&    type != NMXP_MSG_READY) {
475

476
477
478
			/* Process a packet and return value in NMXP_DATA_PROCESS structure */
			pd = nmxp_processCompressedData(buffer, length, channelList_subset, NETCODE_OR_CURRENT_NETWORK);
			nmxp_data_trim(pd, params.start_time, params.end_time, 0);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
479

480
481
482
			/* To prevent to manage a packet with zero sample after nmxp_data_trim() */
			if(pd->nSamp > 0) {

483
484
485
486
			/* Log contents of last packet */
			if(params.flag_logdata) {
			    nmxp_data_log(pd);
			}
487

488
489
			/* Set cur_chan */
			cur_chan = nmxp_chan_lookupKeyIndex(pd->key, channelList_subset);
490
491
492
			if(i_chan != cur_chan) {
			    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_ANY, "i_chan != cur_chan  %d != %d!\n", i_chan, cur_chan);
			}
493

494
			/* Management of gaps */
495
496
			if(!channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
			    channelList_Seq[cur_chan].significant = 1;
497
			} else {
498
499
500
			    if(channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
				if(nmxptool_check_and_log_gap(pd->time, channelList_Seq[cur_chan].last_time, GAP_TOLLERANCE, pd->station, pd->channel)) {
				    channelList_Seq[cur_chan].x_1 = 0;
501
502
503
				    nmxp_data_to_str(str_pd_time, pd->time);
				    nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_EXTRA, "%s.%s x0 set to zero at %s!\n", pd->station, pd->channel, str_pd_time);
				}
504
			    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
505
			}
506
507
			if(channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
			    channelList_Seq[cur_chan].last_time = pd->time + ((double) pd->nSamp / (double) pd->sampRate);
508
			}
509

510
#ifdef HAVE_LIBMSEED
511
512
513
514
			/* Write Mini-SEED record */
			if(params.flag_writeseed) {
			    nmxptool_write_miniseed(pd);
			}
515
#endif
516

517
#ifdef HAVE___SRC_SEEDLINK_PLUGIN_C
518
519
520
521
			/* Send data to SeedLink Server */
			if(params.flag_slink) {
			    nmxptool_send_raw_depoch(pd);
			}
522
523
#endif

524
525
526
527
528
529
#ifdef HAVE_EARTHWORMOBJS
			if(params.ew_configuration_file) {
			    nmxptool_ew_nmx2ew(pd);
			}
#endif

530
531
532
533
534
535
536
537
			if(params.flag_writefile  &&  outfile) {
			    /* Write buffer to the output file */
			    if(outfile && buffer && length > 0) {
				int32_t length_int = length;
				nmxp_data_swap_4b((int32_t *) &length_int);
				fwrite(&length_int, sizeof(length_int), 1, outfile);
				fwrite(buffer, length, 1, outfile);
			    }
538
			}
539

540
541
			}

542
543
			/* Store x_1 */
			if(pd->nSamp > 0) {
544
			    channelList_Seq[cur_chan].x_1 = pd->pDataPtr[pd->nSamp-1];
545
			}
546

547
548
549
550
551
			/* Free pd->buffer */
			if(pd->buffer) {
			    free(pd->buffer);
			    pd->buffer = NULL;
			}
552

553
554
			/* Receive Data */
			ret = nmxp_receiveMessage(naqssock, &type, &buffer, &length, 0, &recv_errno);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
555
			/* nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "ret = %d, type = %d\n", ret, type); */
556
		    }
557

558
559
560
561
562
		    if(params.flag_writefile  &&  outfile) {
			/* Close output file */
			fclose(outfile);
			outfile = NULL;
		    }
563

564
#ifdef HAVE_LIBMSEED
565
566
567
568
569
		    if(params.flag_writeseed  &&  data_seed.outfile_mseed) {
			/* Close output Mini-SEED file */
			fclose(data_seed.outfile_mseed);
			data_seed.outfile_mseed = NULL;
		    }
570
#endif
571

572
573
		}
		i_chan++;
574
	    }
575
	    /* DAP Step 7: Repeat steps 5 and 6 for each data request */
576

577
578
579
580
581
582
583
584
585
586
	    if(params.delay > 0) {
		time_to_sleep = (params.end_time - params.start_time) - (time(NULL) - (params.start_time + params.delay + span_interval));
		if(time_to_sleep >= 0) {
		    nmxp_sleep(time_to_sleep);
		} else {
		    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "time to sleep %d sec.\n", time_to_sleep);
		    nmxp_sleep(3);
		}
		params.start_time = params.end_time;
		params.end_time = params.start_time + span_interval;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
587
	    } else {
588
		exitdapcondition = 0;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
589
	    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
590

591

592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
#ifdef HAVE_EARTHWORMOBJS
	    if(params.ew_configuration_file) {

		/* Check if we are being asked to terminate */
		if( nmxptool_ew_check_flag_terminate() ) {
		    logit ("t", "nmxptool terminating on request\n");
		    nmxptool_ew_send_error(NMXPTOOL_EW_ERR_TERMREQ);
		    exitdapcondition = 0;
		    times_flow = TIMES_FLOW_EXIT;
		}

		/* Check if we need to send heartbeat message */
		nmxptool_ew_send_heartbeat_if_needed();

	    }
#endif

609
	} /* END while(exitdapcondition) */
610

611
612
613
614
615
616
617
618
619
620
	/* DAP Step 8: Send a Terminate message (optional) */
	nmxp_sendTerminateSubscription(naqssock, NMXP_SHUTDOWN_NORMAL, "Bye!");

	/* DAP Step 9: Close the socket */
	nmxp_closeSocket(naqssock);

	/* ************************************************************ */
	/* End subscription protocol "DATA ACCESS PROTOCOL" version 1.0 */
	/* ************************************************************ */

621
	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "End DAP Flow.\n");
622

Matteo Quintiliani's avatar
Matteo Quintiliani committed
623
    } else {
624

625
626
	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "Begin PDS Flow.\n");

627
628
	if(params.stc == -1) {

629

630
631
632
633
	    if(params.flag_logdata) {
		p_func_pd[n_func_pd++] = nmxptool_print_seq_no;
	    }

634
635
636
637
638
639
640
641
642
643
644
645
646
#ifdef HAVE_LIBMSEED
	    /* Write Mini-SEED record */
	    if(params.flag_writeseed) {
		p_func_pd[n_func_pd++] = nmxptool_write_miniseed;
	    }
#endif

#ifdef HAVE___SRC_SEEDLINK_PLUGIN_C
	    /* Send data to SeedLink Server */
	    if(params.flag_slink) {
		p_func_pd[n_func_pd++] = nmxptool_send_raw_depoch;
	    }
#endif
Matteo Quintiliani's avatar
Matteo Quintiliani committed
647
648
649
650
651
652
653

#ifdef HAVE_EARTHWORMOBJS
	    if(params.ew_configuration_file) {
		p_func_pd[n_func_pd++] = nmxptool_ew_nmx2ew;
	    }
#endif

654
	}
655

656
657
658
659
660
661
662
663
664
	/* ************************************************************* */
	/* Start subscription protocol "PRIVATE DATA STREAM" version 1.4 */
	/* ************************************************************* */

	/* PDS Step 1: Open a socket */
	naqssock = nmxp_openSocket(params.hostname, params.portnumberpds);

	if(naqssock == NMXP_SOCKET_ERROR) {
	    return 1;
665
666
	}

667
668
669
670
671
672
673
674
675
676
677
678
679
	/* PDS Step 2: Send a Connect */
	if(nmxp_sendConnect(naqssock) != NMXP_SOCKET_OK) {
	    printf("Error on sendConnect()\n");
	    return 1;
	}

	/* PDS Step 3: Receive ChannelList */
	if(nmxp_receiveChannelList(naqssock, &channelList) != NMXP_SOCKET_OK) {
	    printf("Error on receiveChannelList()\n");
	    return 1;
	}

	/* Get a subset of channel from arguments */
680
	channelList_subset_waste = nmxp_chan_subset(channelList, NMXP_DATA_TIMESERIES, params.channels, CURRENT_NETWORK);
681
682
683


	/* PDS Step 4: Send a Request Pending (optional) */
684
685


686
687
688
689
690
691
	/* PDS Step 5: Send AddChannels */
	/* Request Data */
	nmxp_sendAddTimeSeriesChannel(naqssock, channelList_subset, params.stc, params.rate, (params.flag_buffered)? NMXP_BUFFER_YES : NMXP_BUFFER_NO);

	/* PDS Step 6: Repeat until finished: receive and handle packets */

Matteo Quintiliani's avatar
Matteo Quintiliani committed
692
693
694
695
#ifdef HAVE_LIBMSEED
	if(params.flag_writeseed) {
	    /* Open output Mini-SEED file */
	    sprintf(data_seed.filename_mseed, "%s.realtime.miniseed",
696
		    CURRENT_NETWORK);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
697
698
699

	    data_seed.outfile_mseed = fopen(data_seed.filename_mseed, "w");
	    if(!data_seed.outfile_mseed) {
700
		nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_EXTRA, "Can not to open file %s!", data_seed.filename_mseed);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
701
	    } else {
702
		nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_EXTRA, "Opened file %s!\n", data_seed.filename_mseed);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
703
704
705
706
	    }
	}
#endif

Matteo Quintiliani's avatar
Matteo Quintiliani committed
707
708
709
	// TODO
	exitpdscondition = 1;

710
	skip_current_packet = 0;
711

Matteo Quintiliani's avatar
Matteo Quintiliani committed
712
	while(exitpdscondition) {
Matteo Quintiliani's avatar
Matteo Quintiliani committed
713

714
	    /* Process Compressed or Decompressed Data */
Matteo Quintiliani's avatar
Matteo Quintiliani committed
715
	    pd = nmxp_receiveData(naqssock, channelList_subset, NETCODE_OR_CURRENT_NETWORK, params.timeoutrecv, &recv_errno);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
716

717
718
719
720
721
722
723
724
725
726
727
	    if(!pd) {
		pd_null_count++;
		if((pd_null_count * params.timeoutrecv) >= timeoutrecv_warning) {
		    nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_ANY, "Received %d times a null packet. (%d sec.)\n",
			    pd_null_count, pd_null_count * params.timeoutrecv);
		    pd_null_count = 0;
		}
	    } else {
		pd_null_count = 0;
	    }

728
	    if(recv_errno == 0) {
Matteo Quintiliani's avatar
Matteo Quintiliani committed
729
730
731
		// TODO
		exitpdscondition = 1;
	    } else {
732
733
734
735
736
		if(recv_errno == EWOULDBLOCK) {
		    // TODO
		    exitpdscondition = 1;
		} else {
		    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_CONNFLOW, "Error receiving data. pd=%p recv_errno=%d\n", pd, recv_errno);
737
738

#ifdef HAVE_EARTHWORMOBJS
739
740
741
		    if(params.ew_configuration_file) {
			nmxptool_ew_send_error(NMXPTOOL_EW_ERR_RECVDATA);
		    }
742
#endif
743
744
		    exitpdscondition = 0;
		}
Matteo Quintiliani's avatar
Matteo Quintiliani committed
745
	    }
746

747
748
749
750
751
	    if(pd) {
		/* Set cur_chan */
		cur_chan = nmxp_chan_lookupKeyIndex(pd->key, channelList_subset);
	    }

Matteo Quintiliani's avatar
Matteo Quintiliani committed
752
	    /* Log contents of last packet */
753
754
755
	    if(params.flag_logdata) {
		nmxp_data_log(pd);
	    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
756

757
	    skip_current_packet = 0;
758
	    if(pd &&
759
760
		    (params.statefile  ||  params.buffered_time) &&
		    ( params.timeoutrecv <= 0 )
761
	      )	{
762
763
		if(params.statefile && channelList_Seq[cur_chan].after_start_time > 0.0) {
		    cur_after_start_time = channelList_Seq[cur_chan].after_start_time;
764
765
766
767
768
769
		} else if(params.buffered_time) {
		    cur_after_start_time = params.buffered_time;
		} else {
		    cur_after_start_time = DEFAULT_BUFFERED_TIME;
		}
		nmxp_data_to_str(cur_after_start_time_str, cur_after_start_time);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
770
		nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_PACKETMAN, "cur_chan %d, cur_after_start_time %f, cur_after_start_time_str %s\n", cur_chan, cur_after_start_time, cur_after_start_time_str);
771
772
773
		if(pd->time + ((double) pd->nSamp / (double) pd->sampRate) >= cur_after_start_time) {
		    if(pd->time < cur_after_start_time) {
			int first_nsample_to_remove = (cur_after_start_time - pd->time) * (double) pd->sampRate;
774
775
776
777
			/* Remove the first sample in order avoiding overlap  */
			first_nsample_to_remove++;
			if(pd->nSamp > first_nsample_to_remove) {
			    pd->nSamp -= first_nsample_to_remove;
778
			    pd->time = cur_after_start_time;
779
780
781
782
783
784
785
786
787
788
789
			    pd->pDataPtr += first_nsample_to_remove;
			    pd->x0 = pd->pDataPtr[0];
			} else {
			    skip_current_packet = 1;
			}
		    }
		} else {
		    skip_current_packet = 1;
		}
	    }

Matteo Quintiliani's avatar
Matteo Quintiliani committed
790
	    if(!skip_current_packet) {
791

792
793
		/* Manage Raw Stream */
		if(params.stc == -1) {
Matteo Quintiliani's avatar
Matteo Quintiliani committed
794

795
796
		    /* cur_char is computed only for pd != NULL */
		    if(pd) {
797
798
			nmxp_raw_stream_manage(&(channelList_Seq[cur_chan].raw_stream_buffer), pd, p_func_pd, n_func_pd);
			channelList_Seq[cur_chan].last_time_call_raw_stream = nmxp_data_gmtime_now();
799
		    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
800

801
802
803
804
805
		    /* Check timeout for other channels */
		    if(params.timeoutrecv > 0) {
			exitpdscondition = 1;
			to_cur_chan = 0;
			while(to_cur_chan < channelList_subset->number) {
806
807
			    timeout_for_channel = nmxp_data_gmtime_now() - channelList_Seq[to_cur_chan].last_time_call_raw_stream;
			    if(channelList_Seq[to_cur_chan].last_time_call_raw_stream != 0
808
809
810
				    && timeout_for_channel >= params.timeoutrecv) {
				nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_DOD, "Timeout for channel %s (%d sec.)\n",
					channelList_subset->channel[to_cur_chan].name, timeout_for_channel);
811
812
				nmxp_raw_stream_manage(&(channelList_Seq[to_cur_chan].raw_stream_buffer), NULL, p_func_pd, n_func_pd);
				channelList_Seq[to_cur_chan].last_time_call_raw_stream = nmxp_data_gmtime_now();
813
814
			    }
			    to_cur_chan++;
Matteo Quintiliani's avatar
Matteo Quintiliani committed
815
816
			}
		    }
817
818

		} else {
819
820
821

		    if(pd) {
			/* Management of gaps */
822
823
			if(!channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
			    channelList_Seq[cur_chan].significant = 1;
824
			} else {
825
826
827
			    if(channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
				if(nmxptool_check_and_log_gap(pd->time, channelList_Seq[cur_chan].last_time, GAP_TOLLERANCE, pd->station, pd->channel)) {
				    channelList_Seq[cur_chan].x_1 = 0;
828
829
830
831
832
				    nmxp_data_to_str(str_pd_time, pd->time);
				    nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_EXTRA, "%s.%s x0 set to zero at %s!\n", pd->station, pd->channel, str_pd_time);
				}
			    }
			}
833
834
			if(channelList_Seq[cur_chan].significant && pd->nSamp > 0) {
			    channelList_Seq[cur_chan].last_time = pd->time + ((double) pd->nSamp / (double) pd->sampRate);
835
			}
836

Matteo Quintiliani's avatar
Matteo Quintiliani committed
837
838

#ifdef HAVE_LIBMSEED
839
840
841
842
			/* Write Mini-SEED record */
			if(params.flag_writeseed) {
			    nmxptool_write_miniseed(pd);
			}
Matteo Quintiliani's avatar
Matteo Quintiliani committed
843
844
#endif

845
#ifdef HAVE___SRC_SEEDLINK_PLUGIN_C
846
847
848
849
			/* Send data to SeedLink Server */
			if(params.flag_slink) {
			    nmxptool_send_raw_depoch(pd);
			}
850
#endif
851
852
		    }
		}
853
	    } /* End skip_current_packet condition */
854

Matteo Quintiliani's avatar
Matteo Quintiliani committed
855
	    if(pd) {
856
857
		/* Store x_1 */
		if(pd->nSamp > 0) {
858
		    channelList_Seq[cur_chan].x_1 = pd->pDataPtr[pd->nSamp-1];
859
860
861
862
863
864
		}
		/* Free pd->buffer */
		if(pd->buffer) {
		    free(pd->buffer);
		    pd->buffer = NULL;
		}
Matteo Quintiliani's avatar
Matteo Quintiliani committed
865
	    }
Matteo Quintiliani's avatar
Matteo Quintiliani committed
866

Matteo Quintiliani's avatar
Matteo Quintiliani committed
867
868
869
870
871
872
#ifdef HAVE_EARTHWORMOBJS
	    if(params.ew_configuration_file) {

		/* Check if we are being asked to terminate */
		if( nmxptool_ew_check_flag_terminate() ) {
		    logit ("t", "nmxptool terminating on request\n");
873
		    nmxptool_ew_send_error(NMXPTOOL_EW_ERR_TERMREQ);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
874
875
876
877
878
879
880
881
882
		    exitpdscondition = 0;
		}

		/* Check if we need to send heartbeat message */
		nmxptool_ew_send_heartbeat_if_needed();

	    }
#endif

883
	} /* End main PDS loop */
884

Matteo Quintiliani's avatar
Matteo Quintiliani committed
885
886
	/* Flush raw data stream for each channel */
	flushing_raw_data_stream();
887

Matteo Quintiliani's avatar
Matteo Quintiliani committed
888
889
890
891
892
893
894
#ifdef HAVE_LIBMSEED
	if(params.flag_writeseed  &&  data_seed.outfile_mseed) {
	    /* Close output Mini-SEED file */
	    fclose(data_seed.outfile_mseed);
	}
#endif

895
896
897
898
899
900
901
902
903
904
905

	/* PDS Step 7: Send Terminate Subscription */
	nmxp_sendTerminateSubscription(naqssock, NMXP_SHUTDOWN_NORMAL, "Good Bye!");

	/* PDS Step 8: Close the socket */
	nmxp_closeSocket(naqssock);

	/* *********************************************************** */
	/* End subscription protocol "PRIVATE DATA STREAM" version 1.4 */
	/* *********************************************************** */

906
907
908
	nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CONNFLOW, "End PDS Flow.\n");

    }
909

910
911
912
    if(params.interval == DEFAULT_INTERVAL_INFINITE) {
	times_flow++;
    } else {
913
	times_flow = TIMES_FLOW_EXIT;
914
    }
915

916
917
918
919
    if(params.statefile) {
	save_channel_states(channelList_subset, channelList_Seq);
    }

920
    }
921

922
923
924
925
926
927
#ifdef HAVE_EARTHWORMOBJS
	if(params.ew_configuration_file) {
	    nmxptool_ew_detach();
	}
#endif

928
#ifdef HAVE_LIBMSEED
929
930
931
932
    if(*msr_list_chan) {
	for(i_chan = 0; i_chan < channelList_subset->number; i_chan++) {
	    if(msr_list_chan[i_chan]) {
		msr_free(&(msr_list_chan[i_chan]));
933
934
	    }
	}
935
    }
936
937
#endif

938
    for(i_chan = 0; i_chan < channelList_subset->number; i_chan++) {
939
	nmxp_raw_stream_free(&(channelList_Seq[i_chan].raw_stream_buffer));
940
    }
941

942
943
    if(channelList_Seq) {
	free(channelList_Seq);
944
    }
945

946
947
948
949
    /* This has to be tha last */
    if(channelList_subset) {
	free(channelList_subset);
    }
950

951
    return 0;
952
} /* End MAIN */
953
954


955
956
957
#define MAX_LEN_FILENAME 4096
#define NMXP_STR_STATE_EXT ".nmxpstate"

958
static void save_channel_states(NMXP_CHAN_LIST_NET *chan_list, NMXPTOOL_CHAN_SEQ *chan_list_seq) {
959
960
961
962
    int to_cur_chan;
    char last_time_str[30];
    char raw_last_sample_time_str[30];
    char state_line_str[1000];
963
    FILE *fstatefile = NULL;
964
    char statefilefilename[MAX_LEN_FILENAME] = "";
965

966
    if(params.statefile) {
967
968
969
	strncpy(statefilefilename, params.statefile, MAX_LEN_FILENAME);
	strncat(statefilefilename, NMXP_STR_STATE_EXT, MAX_LEN_FILENAME);
	fstatefile = fopen(statefilefilename, "w");
970
	if(fstatefile == NULL) {
971
	    nmxp_log(NMXP_LOG_ERR, NMXP_LOG_D_ANY, "Unable to write channel states into %s!\n", statefilefilename);
Matteo Quintiliani's avatar
Matteo Quintiliani committed
972
	} else {
973
	    nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_ANY, "Writing channel states into %s!\n", statefilefilename);
974
	}
975

976
977
978
	/* Save state for each channel */
	// if(params.stc == -1)
	to_cur_chan = 0;
979
980
981
	while(to_cur_chan < chan_list->number) {
	    nmxp_data_to_str(last_time_str, chan_list_seq[to_cur_chan].last_time);
	    nmxp_data_to_str(raw_last_sample_time_str, chan_list_seq[to_cur_chan].raw_stream_buffer.last_sample_time);
982
983
	    sprintf(state_line_str, "%d %s %s %s",
		    chan_list->channel[to_cur_chan].key,
984
		    chan_list->channel[to_cur_chan].name,
985
986
		    last_time_str,
		    raw_last_sample_time_str
987
988
989
		   );
	    nmxp_log(NMXP_LOG_NORM, NMXP_LOG_D_CHANNEL, "%s\n", state_line_str);
	    if(fstatefile) {
990
		fprintf(fstatefile, "%s\n", state_line_str);
991
992
993
		if( (chan_list_seq[to_cur_chan].last_time != 0) || (chan_list_seq[to_cur_chan].raw_stream_buffer.last_sample_time != -1.0) ) {
		    nmxp_log(NMXP_LOG_WARN, NMXP_LOG_D_ANY, "%s %d %d %f %f\n", state_line_str, to_cur_chan, chan_list->channel[to_cur_chan].key,
			    chan_list_seq[to_cur_chan].last_time, chan_list_seq[to_cur_chan].raw_stream_buffer.last_sample_time);
994
995
		} else {