#!/bin/bash

# Author: Henrik Virta, Finnish Meteorological Institute (henrik.virta@fmi.fi)
# Update SH 18.12.2025: Added HCHO and SIF)

#### SCRIPT FOR DOWNLOADING GRIDDED L3 TROPOMI FILES FROM KAT S3 SERVER
#
# Syntax: bash download_l3_s3tropomi.sh BUCKET TIMEPERIOD STARTTIME ENDTIME
# BUCKET: no2/o3/co/ch4/so2-(nrti-)l3
# TIMEPERIOD: daily/monthly/yearly
# STARTTIME: YYYY-MM-DD (inclusive)
# ENDTIME: YYYY-MM-DD (exclusive)
#
# Example usage: bash download_l3_s3tropomi.sh no2-l3 daily 2020-06-01 2020-07-01
#


# Script makes use of GNU coreutils commands like "date" rather than their BSD counterparts.
# Such commands are by default prefixed with a "g" (e.g. "gdate") if installed via Homebrew,
# but can be set to override the BSD ones with a modified PATH.
if command -v gdate &> /dev/null; then
    export PATH="/usr/local/opt/coreutils/libexec/gnubin:${PATH}"
fi


# Interpret input parameters
s3bucket="$1"
timeperiod="$2"
startdate="$3"
enddate="$4"


# Check that name of S3 bucket is correct
if [[ ! "$s3bucket" =~ ^(no2|o3|co|ch4|so2|hcho|sif)-(nrti-l3|l3)$ ]]; then
    echo "Incorrect S3 bucket name! Syntax: no2/co/o3/ch4/so2-(nrti-)l3"
    exit 1
fi

# Check that time period is correct
if [[ ! "$timeperiod" =~ ^(daily|monthly|yearly)$ ]]; then
    echo "Incorrect time period name! Options: daily/monthly/yearly"
    exit 1
fi

# Check date syntaxes
dateregex='^20[0-9]{2}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'
if ! [[ "$startdate" =~ $dateregex && "$enddate" =~ $dateregex ]]; then
    echo "Incorrect date format! Syntax: 20YY-MM-DD"
    exit 1
fi

# Ensure end date is greater than start date
if [[ ! "$enddate" > "$startdate" ]]; then
    echo "End date must be greater than start date!"
    exit 1
fi


# Save list of currently available files
echo "Saving file list for s3://tropomi-${s3bucket}/"
if which s3cmd >/dev/null; then
until s3cmd ls "s3://tropomi-${s3bucket}/" > "s3ls_tropomi-${s3bucket}.txt"; do
    echo "S3CMD: Listing of bucket contents failed! Retrying in 1 minute..."
    sleep 1m
done
else
echo "ERROR: s3cmd executable does not exist."
exit 1
fi

# Convert given date range to correct format and find all matching files
case "$timeperiod" in
    daily)
    startdate=$(date -d "$startdate" -u +"%Y%m%d")
    enddate=$(date -d "$enddate" -u +"%Y%m%d")
    matches=$(awk -v FIRST="$startdate" -v LAST="$enddate" '$4 ~ /_dailycomposite_.+\.nc\.gz/ && substr($4,length($4)-13,8) >= FIRST && substr($4,length($4)-13,8) < LAST { print $4 }' "s3ls_tropomi-${s3bucket}.txt")
    ;;
    
    monthly)
    startdate=$(date -d "$startdate" -u +"%Y%m")
    enddate=$(date -d "$enddate" -u +"%Y%m")
    matches=$(awk -v FIRST="$startdate" -v LAST="$enddate" '$4 ~ /_monthlycomposite_.+\.nc\.gz/ && substr($4,length($4)-11,6) >= FIRST && substr($4,length($4)-11,6) < LAST { print $4 }' "s3ls_tropomi-${s3bucket}.txt")
    ;;
    
    yearly)
    startdate=$(date -d "$startdate" -u +"%Y")
    enddate=$(date -d "$enddate" -u +"%Y")
    matches=$(awk -v FIRST="$startdate" -v LAST="$enddate" '$4 ~ /_yearlycomposite_.+\.nc\.gz/ && substr($4,length($4)-9,4) >= FIRST && substr($4,length($4)-9,4) < LAST { print $4 }' "s3ls_tropomi-${s3bucket}.txt")
    ;;
esac


# Show number of results
if [[ -z "$matches" ]]; then
    echo "Found 0 results!"
    exit 0
else
    numres=$(echo "$matches" | wc -l)
    echo "Found ${numres} results. Download starting..."
fi

# Create destination folder if it doesn't exist yet
destfolder="S3_${s3bucket}"
if [ ! -d $destfolder ]; then
    mkdir $destfolder
fi

# Download all matches
for match in $matches; do
    #echo $match
    until s3cmd -f get $match "${destfolder}/"; do
        echo "S3CMD: Download failed. Retrying in 1 minute..."
        sleep 1m
    done
    
    # Decompress downloaded L3 files. This section can be commented out to save hard drive space
    if [[ "$match" == *".nc.gz" ]]; then
        name=$(echo "$match" | awk -F'/' '{ print $NF }')
        gzip -df "${destfolder}/${name}"
    fi
done

echo "Finished!"