''' Program B33_Wx_Station_Data_Download This program is designed to download image and numerical data from the NASA GSFC B33 Website (https://har.gsfc.nasa.gov/index.php?section=73) Inputs: None Outputs: None Notes: 1) This program is designed solely intended to download the daily numerical weather station data (ASCII_WX_STATION) and Quicklook Imagery into a user-specified data directory that parallels the same structure as the weather station website 2) This code was built and tested with Python 3.4 under Ubuntu and Windows 10 Operating Systems. Mac OS was untested, but may still work. This code used standard python modules, but was tested for Anaconda python (https://www.anaconda.com/download) 3) The tqdm module is optional, but does provide more informative download progress information History: 2024-02-08: Original script written by Stephen D. Nicholls 2024-10-13: Updated script to include progress bars via the tqdm module 2025-06-25: Updated script to work with either tqdm for making progress bar or to default to use wget ''' ################################################ # Loading required python modules for this program try: from tqdm import * USE_TQDM = 'Y' except: USE_TQDM = 'N' # Imports utlilty for showing download status bars import datetime as dt # Imports datetime utility import glob # Imports glob function for filename searches import numpy as np # Imports numerical python package import os # Imports operating system commands package import requests # Imports internet data access utility import shutil # Imports high level file operation functions import sys # Imports system commands utility ################################################ start = dt.datetime.now() # Sets initial start time for program #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> ##### START USER SETTINGS ##### USER_DATA_DIR = os.getcwd() # Sets data download directory as current directory location #USER_DATA_DIR = '/home/snicholl/B33_WX_Station_Data' # Sets data download directory as custom location SDATE = dt.datetime(2024,2,7) # Starting Date in Python datetime format (yyyy,mm,dd) EDATE = dt.datetime(2024,2,9) # Ending Date in Python datetime format (yyyy,mm,dd) DOWNLOAD_QLOOKS = 'N' # Download Weather Station Quicklook imagery DOWNLOAD_DATA = 'Y' # Download Weather Station Data # Subset of QLOOK Hours Requested QLOOK_TIMES = np.array([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,21,22,23]) QLOOK_TIMES = np.array([0,6,12,18]) # User-specified QLOOK IMAGERY types QLOOKS_REQUESTED = ['QLOOK_Hourly','QLOOK_Analysis','QLOOK_All_Sky_MRR','QLOOK_Ceil_MRR','QLOOK_All_Sky_Ceil'] # User-specified DATA types DATA_REQUESTED =['All-Sky_Image','All-Sky_Unwrap','Meteo','Present_Weather','Precip_Pluvio','All_Data_Common_Time','Pyranometer','Wind_Sonic','Wind_Young_Prop','CL51_L2_Ceil','CL61_L2_Ceil'] DATA_REQUESTED =['Meteo','Present_Weather','Precip_Pluvio','All_Data_Common_Time'] #DATA_REQUESTED =['CL51_L2_Ceil','CL61_L2_Ceil'] #QLOOKS_REQUESTED = ['QLOOK_All_Sky_MRR'] #DATA_REQUESTED = ['Meteo','Present_Weather'] ''' ALL SUPPORTED OPTIONS AS OF 21 FEB 2024 Quick look imagery (archived hourly): QLOOK_Hourly = Short-term weather station plots (a few hours) QLOOK_Analysis = Longer-term weather station plots (a few days) QLOOK_All_Sky_MRR = Short-term plot showcasing all-sky camera imagery and the microrain radar (MRR) data QLOOK_Ceil_MRR = Short-term plot showcasing Ceilometer (CL51 or CL61) and the microrain radar (MRR) data QLOOK_All_Sky_Ceil = Short-term plot showcasing All-sky camera imagery with Ceilometer (CL51 or CL61) data Data products All-Sky-Image: Full-sky hemispheric imagery produced roughly once per minute and archived into a daily tar.gz file All-Sky-Unwrap: Full-sky unswrapped imagery produced roughly once per minute and archived into a daily tar.gz file CL51_L2_Ceil: Viasala CL51 Backscatter Ceilometer Level 2 Data averaged once per 16 seconds in netcdf format CL61_L2_Ceil: Viasala CL61 Polarized Ceilometer Level 2 Data averaged once per minute in netcdf format Meteo: Standard meteorology data (Temp, Dewpoint, Pressure) recorded once per minute Present_Weather: Visibility and preicipitation type data recorded once per minute Precip_Pluvio: Precipitation data from Pluvio weighing bucket rain guage recorded once per minute All_Data_Common_Time: Merged all weather station sensor data file(Meteo, Pluvio, Present weather, and winds) recorded once per minute Pyranometer: Solar irradiance data recorded once per minute Wind_Sonic: 3-D wind data from campbell scientific sonic anemometer, recorded once every 0.1 seconds Wind_Young_Prop: 2-D wind data from Young propeller anemometer, recorded once every 10 seconds ''' ##### END USER SETTINGS ##### #<<<<<<<<<<<<<<<<<<<<<<<<<<<< ############################ ############################ ##### START MAIN PROCESSING CODE ###### weather_station_website_top_dir = 'https://har.gsfc.nasa.gov/storm/WEATHER' # Top level directory for B33 weather station website data download_filename_list = 'GSFC_B33_Weather_Station_Download_List.txt' # Name of download filelist for wget command # Lists of all available image and dataset options for B33 weather station data ALL_QLOOK_OPTS = ['QLOOK_Hourly', 'QLOOK_Analysis', 'QLOOK_All_Sky_Ceil', 'QLOOK_All_Sky_MRR', 'QLOOK_Ceil_MRR'] ALL_WX_DATA_OPTS = ['All-Sky_Image', 'All-Sky_Unwrap', 'Meteo', 'Present_Weather', 'Precip_Pluvio', 'All_Data_Common_Time', 'Pyranometer', 'Wind_Sonic', 'Wind_Young_Prop','CL51_L2_Ceil','CL61_L2_Ceil'] # Checks user inputs for correctness # QLOOK Image Options for copt in QLOOKS_REQUESTED: if copt in ALL_QLOOK_OPTS: pass else: print ('ERROR: QLOOK_REQUESTED Option ('+copt+') is invalid, please make sure all options are valid and try again') sys.exit() # Numerical and Camera Image Options for copt in DATA_REQUESTED: if copt in ALL_WX_DATA_OPTS: pass else: print ('ERROR: DATA_REQUESTED Option ('+copt+') is invalid, please make sure all options are valid and try again') sys.exit() # Python dictionary containing dataset specific settings WX_STATION_DATA_INFO = { 'QLOOK_Analysis': { 'FILENAME_SEG': 'GSFC_B33_Wx_Analysis', # Filename nomenclature 'FILE_TYPE': '.png', # Data type }, 'QLOOK_Hourly': { 'FILENAME_SEG': 'GSFC_B33_Current_Status', # Filename nomenclature 'FILE_TYPE': '.png', # Data type }, 'QLOOK_All_Sky_Ceil': { 'FILENAME_SEG': 'GSFC_B33_All_Sky_Ceil', # Filename nomenclature 'FILE_TYPE': '.jpg', # Data type }, 'QLOOK_Ceil_MRR': { 'FILENAME_SEG': 'GSFC_B33_Ceil_Radar', # Filename nomenclature 'FILE_TYPE': '.jpg', # Data type }, 'QLOOK_All_Sky_MRR': { 'FILENAME_SEG': 'GSFC_B33_All_Sky_Radar', # Filename nomenclature 'FILE_TYPE': '.jpg', # Data type }, 'All-Sky_Image': { 'FILENAME_SEG': 'All-Sky_Image', # Filename nomenclature 'FILE_TYPE': '.tar.gz', # Data type }, 'All-Sky_Unwrap': { 'FILENAME_SEG': 'All-Sky_Unwrap', # Filename nomenclature 'FILE_TYPE': '.tar.gz', # Data type }, 'CL51_L2_Ceil': { 'FILENAME_SEG': 'CL51_L2_Ceil', # Filename nomenclature 'FILE_TYPE': '.nc', # Data type }, 'CL61_L2_Ceil': { 'FILENAME_SEG': 'CL61_L2_Ceil', # Filename nomenclature 'FILE_TYPE': '.nc', # Data type }, 'Meteo': { 'FILENAME_SEG': 'Meteo', # Filename nomenclature 'FILE_TYPE': '.dat', # Data type }, 'Present_Weather': { 'FILENAME_SEG': 'Present_Weather', # Filename nomenclature 'FILE_TYPE': '.dat', # Data type }, 'Precip_Pluvio': { 'FILENAME_SEG': 'Precip_Pluvio', # Filename nomenclature 'FILE_TYPE': '.dat', # Data type }, 'All_Data_Common_Time': { 'FILENAME_SEG': 'All_Data_Common_Time', # Filename nomenclature 'FILE_TYPE': '.dat', # Data type }, 'Pyranometer': { 'FILENAME_SEG': 'Pyranometer', # Filename nomenclature 'FILE_TYPE': '.dat', # Data type }, 'Wind_Sonic': { 'FILENAME_SEG': 'Wind_Sonic', # Filename nomenclature 'FILE_TYPE': '.dat', # Data type }, 'Wind_Young_Prop': { 'FILENAME_SEG': 'Wind_Young_Prop', # Filename nomenclature 'FILE_TYPE': '.dat', # Data type } } # Downloads Station Quicklook Imagery if DOWNLOAD_QLOOKS == 'Y': # Download Quicklook plots yes (Y) or no (N) #> for copt in QLOOKS_REQUESTED: # Scans through all user-specified Quicklook options #>> print (' ') print ('*********') print (copt) download_filelist = [] # Initialized website filename download list cdate = SDATE # Sets current day of data request while cdate <= EDATE: # Loops through user-specified date range to create download filename list #>>> #print ('Now Creating File Download List of GSFC B33 Weather Station Data on '+dt.datetime.strftime(cdate,'%Y/%m/%d')) #print (copt) # Sets string variables for year, month, and year cyear = dt.datetime.strftime(cdate,'%Y') cmonth = dt.datetime.strftime(cdate,'%m') cday = dt.datetime.strftime(cdate,'%d') # Updates website data location based upon current day being downloaded website_data_dir = weather_station_website_top_dir website_data_dir += '/'+copt+'/'+cyear+'/'+cyear+cmonth+cday #print (website_data_dir) cdata_props = WX_STATION_DATA_INFO[copt] # Obtains variable specific settings from WX_STATION_DATA_INFO # Scans through each user-specified hour and creates Dataset filename for tt in QLOOK_TIMES: if tt < 10: chour = '0'+str(tt) else: chour = str(tt) cfile = cdata_props['FILENAME_SEG']+'_'+cyear+cmonth+cday+'_'+chour+'00'+cdata_props['FILE_TYPE'] # Website and local filename download_filelist.append(website_data_dir+'/'+cfile) # Adds potential website file to download filelist cdate += dt.timedelta(hours=24) # Increments the date by 24 hours #<<< # Downloads all files from the website contained in download_filelist file_count = 1 for url in download_filelist: #>>> cur_file = url.rsplit('/', 1)[-1] # Extracts the local filename from the website url with requests.get(url, stream=True) as r: # If valid data on the website is found, download the data try: r.raise_for_status() # Checks if file exists on the website, if not, condition fails and triggers except condition print ('Exists...downloading...File #'+str(file_count)+'/'+str(len(download_filelist))+' '+url) total_size = int(r.headers.get("content-length", 0)) # Gets total length of the file to be downloaded block_size = 1024 # Sets the blocksize to be written to file if USE_TQDM == 'Y': # Writes website data to file and generates a download progress bar with tqdm(total=total_size, unit="B", unit_scale=True) as progress_bar: with open(cur_file, "wb") as file: for data in r.iter_content(block_size): progress_bar.update(len(data)) file.write(data) else: os.system(f"wget {url}") # If no valid data on the website is found, skips to next file except: print ('Not available on server for this date...skipping...File #'+str(file_count)+'/'+str(len(download_filelist))+' '+url) file_count += 1 # Increments the file counter variable by 1 #<<< #print (' ') #print ('*********') print ('Now Moving Downloaded '+copt+' Data to User-Specified Directory:'+' '+USER_DATA_DIR) cdate = SDATE # Sets current day of data request while cdate <= EDATE: #>>> #Sets string variables for year, month, and year cyear = dt.datetime.strftime(cdate,'%Y') cmonth = dt.datetime.strftime(cdate,'%m') cday = dt.datetime.strftime(cdate,'%d') # Tests for the existence of and creates file directory stucture where files are to be stored local_data_dir = USER_DATA_DIR try: os.stat(local_data_dir) except: os.mkdir(local_data_dir) local_data_dir = os.path.join(local_data_dir,'QLOOK_GSFC_B33_WX_STATION') try: os.stat(local_data_dir) except: os.mkdir(local_data_dir) local_data_dir = os.path.join(local_data_dir,cyear) try: os.stat(local_data_dir) except: os.mkdir(local_data_dir) local_data_dir = os.path.join(local_data_dir,cyear+cmonth+cday) try: os.stat(local_data_dir) except: os.mkdir(local_data_dir) #print (local_data_dir) cdata_props = WX_STATION_DATA_INFO[copt] # Obtains variable specific settings from WX_STATION_DATA_INFO cfile_wc = cdata_props['FILENAME_SEG']+'_'+cyear+cmonth+cday+'**'+cdata_props['FILE_TYPE'] # Sets a filename wildcard for the current dataset cfiles = glob.glob1(USER_DATA_DIR,cfile_wc) # Creates list of all variables names meeting the wildcard # Moves any files in the users' working directory to their final locations if len(cfiles) > 0: for cfile in cfiles: #os.system('mv -f '+cfile+' '+local_data_dir) try: os.remove(os.path.join(local_data_dir,cfile)) except: pass shutil.move(cfile, local_data_dir) else: pass # If no files are found, none are moved cdate += dt.timedelta(hours=24) # Increments current data search date by 24 hours #<<< #<< #< # Downloads Station Quicklook Imagery if DOWNLOAD_DATA == 'Y': # Download data yes (Y) or no (N) #> for copt in DATA_REQUESTED: # Scans through all user-specified Data options #>> print (' ') print ('*********') print (copt) download_filelist = [] # Initialized website filename download list cdate = SDATE # Sets current day of data request while cdate <= EDATE: # Loops through user-specified date range to create download filename list #>>> #print (' ') #print ('*********') #print ('Now Creating File Download List of GSFC B33 Weather Station Data on '+dt.datetime.strftime(cdate,'%Y/%m/%d')) #print (copt) # Sets string variables for year, month, and year cyear = dt.datetime.strftime(cdate,'%Y') cmonth = dt.datetime.strftime(cdate,'%m') cday = dt.datetime.strftime(cdate,'%d') # Updates website data location based upon current day being downloaded website_data_dir = weather_station_website_top_dir website_data_dir += '/'+'ASCII_WX_STATION_DATA_HISTORICAL'+'/'+cyear+'/'+cyear+cmonth+cday #print (website_data_dir) cdata_props = WX_STATION_DATA_INFO[copt] # Obtains variable specific settings from WX_STATION_DATA_INFO # Scans through each user-specified hour and creates Dataset filename cfile = cyear+cmonth+cday+'_'+cdata_props['FILENAME_SEG']+cdata_props['FILE_TYPE'] # Website and local filename download_filelist.append(website_data_dir+'/'+cfile) # Adds potential website file to download filelist cdate += dt.timedelta(hours=24) # Increments the date by 24 hours #<<< # Downloads all files from the website contained in download_filelist file_count = 1 for url in download_filelist: #>>> cur_file = url.rsplit('/', 1)[-1] # Extracts the local filename from the website url with requests.get(url, stream=True) as r: # If valid data on the website is found, download the data try: r.raise_for_status() # Checks if file exists on the website, if not, condition fails and triggers except condition print ('Exists...downloading...File #'+str(file_count)+'/'+str(len(download_filelist))+' '+url) total_size = int(r.headers.get("content-length", 0)) # Gets total length of the file to be downloaded block_size = 1024 # Sets the blocksize to be written to file if USE_TQDM == 'Y': # Writes website data to file and generates a download progress bar with tqdm(total=total_size, unit="B", unit_scale=True) as progress_bar: with open(cur_file, "wb") as file: for data in r.iter_content(block_size): progress_bar.update(len(data)) file.write(data) else: os.system(f"wget {url}") # If no valid data on the website is found, skips to next file except: print ('Not available on server for this date...skipping...File #'+str(file_count)+'/'+str(len(download_filelist))+' '+url) file_count += 1 # Increments the file counter variable by 1 #<<< #print (' ') #print ('*********') print ('Now Moving Downloaded '+copt+' Data to User-Specified Directory:'+' '+USER_DATA_DIR) cdate = SDATE # Sets current day of data request while cdate <= EDATE: #>>> #Sets string variables for year, month, and year cyear = dt.datetime.strftime(cdate,'%Y') cmonth = dt.datetime.strftime(cdate,'%m') cday = dt.datetime.strftime(cdate,'%d') # Tests for the existence of and creates file directory stucture where files are to be stored local_data_dir = USER_DATA_DIR try: os.stat(local_data_dir) except: os.mkdir(local_data_dir) local_data_dir = os.path.join(local_data_dir,'DATA_GSFC_B33_WX_STATION') try: os.stat(local_data_dir) except: os.mkdir(local_data_dir) local_data_dir = os.path.join(local_data_dir,cyear) try: os.stat(local_data_dir) except: os.mkdir(local_data_dir) local_data_dir = os.path.join(local_data_dir,cyear+cmonth+cday) try: os.stat(local_data_dir) except: os.mkdir(local_data_dir) #print (local_data_dir) cdata_props = WX_STATION_DATA_INFO[copt] # Obtains variable specific settings from WX_STATION_DATA_INFO cfile_wc = cfile_wc = cyear+cmonth+cday+'_'+cdata_props['FILENAME_SEG']+cdata_props['FILE_TYPE'] # Sets a filename wildcard for the current dataset cfiles = glob.glob1(USER_DATA_DIR,cfile_wc) # Creates list of all variables names meeting the wildcard # Moves any files in the users' working directory to their final locations if len(cfiles) > 0: for cfile in cfiles: #os.system('mv -f '+cfile+' '+local_data_dir) try: os.remove(os.path.join(local_data_dir,cfile)) except: pass shutil.move(cfile, local_data_dir) else: pass # If no files are found, none are moved cdate += dt.timedelta(hours=24) # Increments current data search date by 24 hours #<<< #<< #< ##### END MAIN PROCESSING CODE ###### print ('Program has completed successfully') print ('Elapsed time is '+str((dt.datetime.now()-start).total_seconds())+' Seconds')