#!/bin/bash

# Author: Henrik Virta, Finnish Meteorological Institute (henrik.virta@fmi.fi)
# Update SH 18.12.2025: Added HCHO and SIF)

#### SCRIPT FOR DOWNLOADING GRIDDED L3 TROPOMI FILES FROM KAT S3 SERVER
#
# Syntax: bash download_l3_s3tropomi.sh BUCKET TIMEPERIOD STARTTIME ENDTIME
# BUCKET: no2/o3/co/ch4/so2-(nrti-)l3
# TIMEPERIOD: daily/monthly/yearly
# STARTTIME: YYYY-MM-DD (inclusive)
# ENDTIME: YYYY-MM-DD (exclusive)
#
# Example usage: bash download_l3_s3tropomi.sh no2-l3 daily 2020-06-01 2020-07-01
#


# Script makes use of GNU coreutils commands like "date" rather than their BSD counterparts.
# Such commands are by default prefixed with a "g" (e.g. "gdate") if installed via Homebrew,
# but can be set to override the BSD ones with a modified PATH.
if command -v gdate &> /dev/null; then
    export PATH="/usr/local/opt/coreutils/libexec/gnubin:${PATH}"
fi


# Interpret input parameters
s3bucket="$1"
timeperiod="$2"
startdate="$3"
enddate="$4"


# Check that name of S3 bucket is correct
if [[ ! "$s3bucket" =~ ^(no2|o3|co|ch4|so2|hcho|sif)-(nrti-l3|l3)$ ]]; then
    echo "Incorrect S3 bucket name! Syntax: no2/co/o3/ch4/so2-(nrti-)l3"
    exit 1
fi

# Check that time period is correct
if [[ ! "$timeperiod" =~ ^(daily|monthly|yearly)$ ]]; then
    echo "Incorrect time period name! Options: daily/monthly/yearly"
    exit 1
fi

# Check date syntaxes
dateregex='^20[0-9]{2}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'
if ! [[ "$startdate" =~ $dateregex && "$enddate" =~ $dateregex ]]; then
    echo "Incorrect date format! Syntax: 20YY-MM-DD"
    exit 1
fi

# Ensure end date is greater than start date
if [[ ! "$enddate" > "$startdate" ]]; then
    echo "End date must be greater than start date!"
    exit 1
fi


# Save list of currently available files
echo "Saving file list for s3://tropomi-${s3bucket}/"
if which s3cmd >/dev/null; then
until s3cmd ls "s3://tropomi-${s3bucket}/" > "s3ls_tropomi-${s3bucket}.txt"; do
    echo "S3CMD: Listing of bucket contents failed! Retrying in 1 minute..."
    sleep 1m
done
else
echo "ERROR: s3cmd executable does not exist."
exit 1
fi

# Convert given date range to correct format and find all matching files
case "$timeperiod" in
    daily)
  if date --version >/dev/null 2>&1; then
    startdate=$(date -d "$startdate" -u +"%Y%m%d")
    enddate=$(date -d "$enddate" -u +"%Y%m%d")
  else
    startdate=$(date -j -u -f "%Y-%m-%d" "$startdate" "+%Y%m%d")
    enddate=$(date -j -u -f "%Y-%m-%d" "$enddate" "+%Y%m%d")
  fi
  matches=$(awk -v FIRST="$startdate" -v LAST="$enddate" '
  $4 ~ /_dailycomposite_[0-9]{8}\.png$/ {
    date = $4
    sub(/^.*_dailycomposite_/, "", date)
    sub(/\.png$/, "", date)

    if (date >= FIRST && date < LAST) print $4
  }
' "s3ls_tropomi-${s3bucket}.txt")
    ;;
    
    monthly)
  if date --version >/dev/null 2>&1; then
    # GNU date (Linux)
    startdate=$(date -d "$startdate" -u +"%Y%m")
    enddate=$(date -d "$enddate" -u +"%Y%m")
  else # BSD date (macOS)
    startdate=$(date -j -u -f "%Y-%m-%d" "$startdate" "+%Y%m")
    enddate=$(date -j -u -f "%Y-%m-%d" "$enddate" "+%Y%m")
  fi
matches=$(awk -v FIRST="$startdate" -v LAST="$enddate" '
  $4 ~ /_monthlycomposite_[0-9]{6}\.png$/ {
    date = $4
    sub(/^.*_monthlycomposite_/, "", date)
    sub(/\.png$/, "", date)

    month = substr(date, 1, 6)

    if (month >= FIRST && month <= LAST) print $4
  }
' "s3ls_tropomi-${s3bucket}.txt")
;;
    
    yearly)
  if date --version >/dev/null 2>&1; then
    startdate=$(date -d "$startdate" -u +"%Y")
    enddate=$(date -d "$enddate" -u +"%Y")
  else
    startdate=$(date -j -u -f "%Y-%m-%d" "$startdate" "+%Y")
    enddate=$(date -j -u -f "%Y-%m-%d" "$enddate" "+%Y")
  fi
 matches=$(awk -v FIRST="$startdate" -v LAST="$enddate" '
  $4 ~ /_yearlycomposite_[0-9]{4}\.png$/ {
    date = $4
    sub(/^.*_yearlycomposite_/, "", date)
    sub(/\.png$/, "", date)

    year = substr(date, 1, 4)

    if (year >= FIRST && year <= LAST) print $4
  }
' "s3ls_tropomi-${s3bucket}.txt")
esac
echo "$startdate"
echo "$enddate"

# Show number of results
if [[ -z "$matches" ]]; then
    echo "Found 0 results!"
    exit 0
else
    numres=$(echo "$matches" | wc -l)
    echo "Found ${numres} results. Download starting..."
fi

# Create destination folder if it doesn't exist yet
destfolder="S3_${s3bucket}"
if [ ! -d $destfolder ]; then
    mkdir $destfolder
fi

# Download all matches
for match in $matches; do
    #echo $match
    until s3cmd -f get $match "${destfolder}/"; do
        echo "S3CMD: Download failed. Retrying in 1 minute..."
        sleep 1m
    done
    
    # Decompress downloaded L3 files. This section can be commented out to save hard drive space
    if [[ "$match" == *".png" ]]; then
        name=$(echo "$match" | awk -F'/' '{ print $NF }')
        gzip -df "${destfolder}/${name}"
    fi
done

echo "Finished!"