#!/bin/bash

# Script to list and download files from a public S3 bucket using curl and awk

# Authors: Henrik Virta and Seppo Hassinen, Finnish Meteorological Institute (henrik.virta@fmi.fi)
# Update: SH, added HCHO and SIF

#### SCRIPT FOR DOWNLOADING GRIDDED L3 TROPOMI FILES FROM KAT S3 SERVER
#
# Syntax: bash download_l3_s3tropomi.sh BUCKET TIMEPERIOD STARTTIME ENDTIME
# BUCKET: no2/o3/co/ch4/so2-(nrti-)l3
# TIMEPERIOD: daily/monthly/yearly
# STARTTIME: YYYY-MM-DD (inclusive)
# ENDTIME: YYYY-MM-DD (exclusive)
#
# Example usage: sh download_l3_s3tropomi.sh no2-l3 daily 2020-06-01 2020-07-01
#
# Requirements: Bash shell. Either GNU Linux system or BSD system (like Mac) with installed ggrep, awk and curl          

# Script makes use of GNU coreutils commands like "date" rather than their BSD counterparts.
# Such commands are by default prefixed with a "g" (e.g. "gdate") if installed via Homebrew,
# but can be set to override the BSD ones with a modified PATH.
if command -v gdate &> /dev/null; then
    export PATH="/usr/local/opt/coreutils/libexec/gnubin:${PATH}"
fi

# Interpret input parameters
s3bucket="$1"
timeperiod="$2"
startdate="$3"
enddate="$4"
bucket_url="https://tropomi-$s3bucket.data.lit.fmi.fi"
continuation_token=""


# Validate inputs (omitted for brevity, keep your existing validation logic here)
# Check that name of S3 bucket is correct
if [[ ! "$s3bucket" =~ ^(no2|o3|co|ch4|so2|hcho|sif)-(nrti-l3|l3)$ ]]; then
    echo "Incorrect S3 bucket name! Syntax: no2/co/o3/ch4/so2/hcho/sif-(nrti-)l3"
    exit 1
fi

# Check that time period is correct
if [[ ! "$timeperiod" =~ ^(daily|monthly|yearly)$ ]]; then
    echo "Incorrect time period name! Options: daily/monthly/yearly"
    exit 1
fi

# Check date syntaxes
dateregex='^20[0-9]{2}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$'
if ! [[ "$startdate" =~ $dateregex && "$enddate" =~ $dateregex ]]; then
    echo "Incorrect date format! Syntax: 20YY-MM-DD"
    exit 1
fi

# Ensure end date is greater than start date
if [[ ! "$enddate" > "$startdate" ]]; then
    echo "End date must be greater than start date!"
    exit 1
fi

# Function to format dates according to the time period
format_dates() {
    case "$1" in
        daily)
        if date --version >/dev/null 2>&1; then
          # GNU date (Linux)
            echo $(date -d "$2" -u +"%Y%m%d")
        else
          # BSD date (macOS)
          echo $(date -jf "%Y-%m-%d" "$2" +"%Y%m%d")
        fi
        ;;
        monthly)
        if date --version >/dev/null 2>&1; then
          # GNU date (Linux)
            echo $(date -d "$2" -u +"%Y%m")
        else
          # BSD date (macOS)
          echo $(date -jf "%Y-%m-%d" "$2" +"%Y%m")
        fi
        ;;
        yearly)
        if date --version >/dev/null 2>&1; then
          # GNU date (Linux)
          echo $(date -d "$2" -u +"%Y")
        else
          # BSD date (macOS)
          echo $(date -jf "%Y-%m-%d" "$2" +"%Y")
        fi
        ;;
    esac
}

# Format start and end dates
formatted_startdate=$(format_dates "$timeperiod" "$startdate")
formatted_enddate=$(format_dates "$timeperiod" "$enddate")

echo "Formatted startdate: $formatted_startdate"
echo "Formatted enddate: $formatted_enddate"

# Fetch bucket contents using curl
#curl -s "$bucket_url" | grep -oP '(?<=<Key>).*?(?=</Key>)' > bucket_contents.txt
temp_file="bucket_contents.txt"

# Try using grep with -P, and if it fails, fall back to a different command or strategy
if grep --version 2>&1 | grep -q BSD; then
    # System has BSD grep
    if which ggrep >/dev/null 2>&1; then
        grep_cmd="ggrep"
    else
        echo "GNU grep (ggrep) not found. Script requires Perl-compatible regular expressions (-P)."
        exit 1
    fi

else
    # GNU grep is available, for systems like macOS
    grep_cmd="grep"
fi


while : ; do
    response_file=$(mktemp)
    if [[ -n $continuation_token ]]; then
        curl -s "$bucket_url?list-type=2&continuation-token=$continuation_token" > "$response_file"
    else
        curl -s "$bucket_url?list-type=2" > "$response_file"
    fi

    # Extract keys and continuation token from response
    # This requires proper XML parsing to reliably extract keys and the token
      $grep_cmd -oP '(?<=<Key>).*?(?=</Key>)' "$response_file" >> "$temp_file"
      continuation_token=$($grep_cmd -oPm1 '(?<=<NextContinuationToken>).*?(?=</NextContinuationToken>)' "$response_file")

    # Break the loop if no continuation token is found
    if [[ -z $continuation_token ]]; then
        break
    fi

    # Cleanup temporary response file
    rm "$response_file"
done

# Filter matching files based on time period and date range
pattern=""
case "$timeperiod" in
    daily)
        pattern="_dailycomposite_.*\\.nc\\.gz"
        matches=$(awk -v pattern="$pattern" -v start="$formatted_startdate" -v end="$formatted_enddate" \
        '$0 ~ pattern && match($0, /dailycomposite_[0-9]{8}/) {
            day = substr($0, RSTART+15, 8);
            if (day >= start && day < end) print;
        }' bucket_contents.txt)
        ;;
    monthly)
        pattern="_monthlycomposite_.*\\.nc\\.gz"
        matches=$(awk -v pattern="$pattern" -v start="$formatted_startdate" -v end="$formatted_enddate" \
        '$0 ~ pattern && match($0, /monthlycomposite_[0-9]{6}/) {
            month = substr($0, RSTART+17, 6);
            if (month >= start && month < end) print;
        }' bucket_contents.txt)
        ;;
    yearly)
        pattern="_yearlycomposite_.*\\\.nc\\\.gz"
        matches=$(awk -v pattern="$pattern" -v start="$formatted_startdate" -v end="$formatted_enddate" \
        '$0 ~ pattern && match($0, /yearlycomposite_[0-9]{4}/) {
            year = substr($0, RSTART+16, 4);
            if (year >= start && year < end) print;
        }' bucket_contents.txt)
        ;;
esac


# Create destination folder
destfolder="S3_${s3bucket}"
mkdir -p "$destfolder"

# Download matched files
echo "$matches" | while read -r match; do
    echo "Downloading $match..."
    curl -s "$bucket_url/$match" -o "${destfolder}/$(basename "$match")"

    # Decompress if necessary, you may comment this out
    if [[ "$match" == *".nc.gz" ]]; then
        gzip -d "${destfolder}/$(basename "$match")"
    fi
done

rm bucket_contents.txt

echo "Finished!"