Commit d796906d authored by Sergio Bruni's avatar Sergio Bruni

ciao

parent a9649325
#!/usr/bin/env python
"""Usage: populateLog.py [-h] [-s] [-a] [-m] [-r root_path] [-w wrong_path] [-b to_be_indexed_path] [-i indexed_path] [-c station_code] [-C station_list_file] [-l list_file] [-H rhost] [-U ruser] [-P rpassword] [-K private_key] [-n net] [-y year] [-o doy] [-d debug] [--verbose] [-e list_excluded]
"""Usage: populateLog.py [-h] [-y start_year] [-Y end_year] [-d start_doy] [-D end_doy] [-c cluster_name] [-p pcf_file] [-H rhost] [-U ruser] [-P rpassword] [-K private_key] [-u debug] [--verbose]
Options:
-h --help Show this screen.
-s --skip_existing Skip anubis check for file already indexed
-a --skip_anubis Skip Anubis check
--version Show version.
-r --root_path path Root path to search files for
-w --wrong_path path Root path where to copy wrong files discarded by indexGD (empty or unreadable)
-b --to_be_indexed_path path Root path where to copy files, processed by indexGD, which were not indexed for any reason
-i --indexed_path path Root path where to copy indexed files
-n --net net_name Specific network
-y --year year Specific year (CCYY)
-o --doy doy Specific doy
-d --debug debug Debug level [default: WARN]
-y --start_year year Start year (YYYY)
-Y --end_year year End year (YYYY)
-d --start_doy doy Start doy (DDD)
-D --end_doy doy End doy (DDD)
-c --cluster_name string Cluster name (string, usually 3-char)
-p --pcf_file filename PCF filename
-u --debug debug Debug level [default: WARN]
-H --rhost host Remote host to scan via sftp
-U --ruser ruser Remote host user
-P --rpassword password Remote host password
-K --private_key key_file Private ssh key for remote access
-c --station_code marker Get only the station with this marker
-C --station_list_file file External file with list of stations to get
-e --list_excluded file Add excluded file in this output file
-m --remove_files Remove files from root_path after elaboration
-l --list_file file Get files to index from file list instead of root path
"""
......@@ -47,11 +39,9 @@ import shutil
import requests
import json
import pysftp
from jsoncomment import JsonComment
from configparser import SafeConfigParser
import sys
from functools import wraps
import georinex
import ntpath
PROF_DATA = {}
......@@ -92,7 +82,7 @@ def clear_prof_data():
global PROF_DATA
PROF_DATA = {}
class GnssIndexer(object):
class RUNbernese(object):
def __new__(cls, args, conf):
#def __new__(cls):
#self = super(GnssIndexer, cls).__new__(cls, args, conf)
......@@ -740,10 +730,10 @@ class GnssIndexer(object):
for my_file in my_files:
self.filter_file(my_file)
class GnssIndexerRemote(GnssIndexer):
class RUNberneseRemote(RUNbernese):
def __new__(cls, args, conf):
#self = super(GnssIndexer, cls).__new__(cls, args, conf)
self = super(GnssIndexer, cls).__new__(cls)
self = super(RUNbernese, cls).__new__(cls)
self.args = args
try:
cnopts = None
......@@ -934,26 +924,21 @@ if __name__ == '__main__':
schema = Schema({
'--verbose': Or(False, True),
'--help': Or(False, True),
'--remove_files': Or(False, True),
'--skip_existing': Or(False, True),
'--skip_anubis': Or(False, True),
'--net': Or(And(str, len),
error = '--net option is mandatory'),
'--year': Or(None, And(str, len)),
'--doy': Or(None, And(str, len)),
'--start_year': Or(And(str, len),
error = '--start_year is mandatory'),
'--start_doy': Or(And(str, len),
error = '--start_doy is mandatory'),
'--cluster_name': Or(And(str, len),
error = '--cluster_name is mandatory'),
'--pcf_file': Or(None, And(str, len),
error='--pcf_file is mandatory'),
'--end_year': Or(None, And(str, len)),
'--end_doy': Or(None, And(str, len)),
'--rhost': Or(None, And(str, len)),
'--ruser': Or(None, And(str, len)),
'--rpassword': Or(None, And(str, len)),
'--private_key': Or(None, And(str, len)),
'--debug': And(str, len),
'--root_path': Or(None, And(str, len)),
'--wrong_path': Or(None, And(str, len)),
'--to_be_indexed_path': Or(None, And(str, len)),
'--indexed_path': Or(None, And(str, len)),
'--list_excluded': Or(None, And(str, len)),
'--station_code': Or(None, And(str, len)),
'--station_list_file': Or(None, And(str, len)),
'--list_file': Or(None, And(str, len))
'--debug': And(str, len)
})
try:
......@@ -971,55 +956,31 @@ if __name__ == '__main__':
print ("At lest one of rpassword or private_key parameters must be provided for remote acces ...")
exit(0)
if args['--station_list_file']:
if not os.path.isfile(args['--station_list_file']):
print("file {} does not exist ".format(args['--station_list_file']))
exit(0)
if args['--station_code']:
print("Parameters --station_list_file and --station_code are not compatible together")
exit(0)
if not args['--end_year']:
args['--end_year'] = args['--start_year']
if not args['--list_file'] and not args['--root_path']:
print("At least one of the two parameters --list_file and --root_path must be provided")
exit(0)
if not args['--end_doy']:
args['--end_doy'] = args['--start_doy']
if args['--list_file']:
if args['--root_path']:
print("Parameters --list_file and --root_path are not compatible together")
exit(0)
if args['--year']:
print("Parameters --list_file and --year are not compatible together")
exit(0)
if args['--doy']:
print("Parameters --list_file and --doy are not compatible together")
exit(0)
if args['--doy']:
args['--doy'] = args['--doy'].zfill(3)
conf_data = parseConfig()
conf_data = {}
#conf_data = parseConfig()
# check configuration params
if not os.path.isfile(conf_data['Programs']['indexGD']):
print("Program {} indicated in configuration param [Programs][indexGD] param does not exists" \
.format(conf_data['Programs']['indexGD']))
exit(3)
if not os.path.isfile(conf_data['Programs']['RunQC']):
print("Program {} indicated in configuration param [Programs][RunQC] param does not exists" \
.format(conf_data['Programs']['RunQC']))
exit(3)
#if not os.path.isfile(conf_data['Programs']['indexGD']):
# print("Program {} indicated in configuration param [Programs][indexGD] param does not exists" \
# .format(conf_data['Programs']['indexGD']))
# exit(3)
if args["--rhost"] is None:
scanner = GnssIndexer(args, conf_data)
s = RUNbernese(args, conf_data)
else:
scanner = GnssIndexerRemote(args, conf_data)
s = RUNberneseRemote(args, conf_data)
if scanner:
scanner.run()
if s:
s.run()
messages = print_prof_data()
for message in messages:
scanner.logger.warn(message)
s.logger.warning(message)
if scanner.tmpdirname and os.path.exists(scanner.tmpdirname):
shutil.rmtree(scanner.tmpdirname)
if s.tmpdirname and os.path.exists(s.tmpdirname):
shutil.rmtree(s.tmpdirname)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment