Инвентаризация бакетов S3

Иногда полезно оценить насколько пакеты S3 актуальны — права доступа по ним, объем, количество файлов, дата создания ближайшего файла.

Предварительно установите и настройте s3cmd

#!/bin/bash
# Buckets names
BUCKETS="bucked1 bucked2 bucked3"

# Temporary file for results
TEMP_FILE=$(mktemp)
SUMMARY_FILE=$(mktemp)

echo "bucket;access_rights;volume;file_count;latest_file_date(MSK)"
echo "------------------------------------------------------------"

TOTAL_BUCKETS=0
TOTAL_OBJECTS=0
TOTAL_SIZE_BYTES=0
EXISTING_BUCKETS=0

for BUCKET in $BUCKETS; do
    TOTAL_BUCKETS=$((TOTAL_BUCKETS + 1))
    echo "Processing: $BUCKET" >&2
    
    # Check if bucket exists
    if ! s3cmd ls s3://$BUCKET/ > /dev/null 2>&1; then
        echo "$BUCKET;bucket does not exist;;;" | tee -a "$TEMP_FILE"
        echo "Status: NOT FOUND" >&2
        echo "" >&2
        continue
    fi
    
    EXISTING_BUCKETS=$((EXISTING_BUCKETS + 1))
    
    # Get ALL ACL lines
    ACL_INFO=$(s3cmd info s3://$BUCKET/ 2>/dev/null)
    
    # Extract ALL ACL lines and combine them
    ACL_LINES=$(echo "$ACL_INFO" | grep "ACL:" | sed 's/.*ACL:[[:space:]]*//')
    
    # Format ACL - join multiple lines with semicolon
    ACL_FORMATTED=$(echo "$ACL_LINES" | tr '\n' ';' | sed 's/;*$//' | sed 's/;/ | /g')
    
    # If no ACL found
    [ -z "$ACL_FORMATTED" ] && ACL_FORMATTED="not available"
    
    # Get all objects
    OBJECTS=$(s3cmd ls --recursive s3://$BUCKET/ 2>/dev/null)
    
    # Count objects
    OBJECT_COUNT=0
    if [ -n "$OBJECTS" ]; then
        OBJECT_COUNT=$(echo "$OBJECTS" | wc -l | tr -d ' ')
    fi
    TOTAL_OBJECTS=$((TOTAL_OBJECTS + OBJECT_COUNT))
    
    # Calculate size
    SIZE_BYTES=0
    if [ -n "$OBJECTS" ]; then
        SIZE_BYTES=$(echo "$OBJECTS" | grep -v "DIR" | awk '{sum+=$3} END{print sum+0}')
    fi
    TOTAL_SIZE_BYTES=$((TOTAL_SIZE_BYTES + SIZE_BYTES))
    
    # Format size
    if [ $SIZE_BYTES -ge 1073741824 ]; then
        VOL_FORMATTED=$(printf "%.2f GB" $(echo "scale=2; $SIZE_BYTES/1073741824" | bc))
    elif [ $SIZE_BYTES -ge 1048576 ]; then
        VOL_FORMATTED=$(printf "%.2f MB" $(echo "scale=2; $SIZE_BYTES/1048576" | bc))
    elif [ $SIZE_BYTES -ge 1024 ]; then
        VOL_FORMATTED=$(printf "%.2f KB" $(echo "scale=2; $SIZE_BYTES/1024" | bc))
    else
        VOL_FORMATTED="$SIZE_BYTES B"
    fi
    
    # Get latest file
    LATEST_FILE=""
    if [ -n "$OBJECTS" ]; then
        LATEST_FILE=$(echo "$OBJECTS" | grep -v "DIR" | sort -k1,1r -k2,2r | head -1)
    fi
    
    if [ -n "$LATEST_FILE" ]; then
        UTC_DATE=$(echo "$LATEST_FILE" | awk '{print $1" "$2}')
        MSK_DATE=$(date -j -v+3H -f "%Y-%m-%d %H:%M" "$UTC_DATE" "+%d.%m.%Y %H:%M" 2>/dev/null || echo "$UTC_DATE")
    else
        MSK_DATE="no files"
    fi
    
    # Output to table
    echo "$BUCKET;$ACL_FORMATTED;$VOL_FORMATTED;$OBJECT_COUNT;$MSK_DATE" | tee -a "$TEMP_FILE"
    
    # Save for summary
    echo "$BUCKET:$OBJECT_COUNT:$SIZE_BYTES" >> "$SUMMARY_FILE"
    
    echo "Status: OK - $OBJECT_COUNT objects, $VOL_FORMATTED" >&2
    echo "" >&2
done

# Display results
echo ""
echo "=== RESULTS ==="
cat "$TEMP_FILE"

echo ""
echo "=== SUMMARY ==="
echo "Total buckets checked: $TOTAL_BUCKETS"
echo "Existing buckets: $EXISTING_BUCKETS"
echo "Non-existent buckets: $((TOTAL_BUCKETS - EXISTING_BUCKETS))"
echo ""

if [ $EXISTING_BUCKETS -gt 0 ]; then
    # Format total size
    if [ $TOTAL_SIZE_BYTES -ge 1073741824 ]; then
        TOTAL_SIZE_FORMATTED=$(printf "%.2f GB" $(echo "scale=2; $TOTAL_SIZE_BYTES/1073741824" | bc))
    elif [ $TOTAL_SIZE_BYTES -ge 1048576 ]; then
        TOTAL_SIZE_FORMATTED=$(printf "%.2f MB" $(echo "scale=2; $TOTAL_SIZE_BYTES/1048576" | bc))
    elif [ $TOTAL_SIZE_BYTES -ge 1024 ]; then
        TOTAL_SIZE_FORMATTED=$(printf "%.2f KB" $(echo "scale=2; $TOTAL_SIZE_BYTES/1024" | bc))
    else
        TOTAL_SIZE_FORMATTED="$TOTAL_SIZE_BYTES B"
    fi
    
    echo "Total objects: $TOTAL_OBJECTS"
    echo "Total size: $TOTAL_SIZE_FORMATTED ($TOTAL_SIZE_BYTES bytes)"
    echo ""
    
    # Show per bucket stats
    echo "Per bucket details:"
    echo "-------------------"
    while IFS=':' read -r bucket count size; do
        if [ "$size" -ge 1048576 ]; then
            bucket_size=$(printf "%.2f MB" $(echo "scale=2; $size/1048576" | bc))
        elif [ "$size" -ge 1024 ]; then
            bucket_size=$(printf "%.2f KB" $(echo "scale=2; $size/1024" | bc))
        else
            bucket_size="$size B"
        fi
        echo "$bucket: $count objects, $bucket_size"
    done < "$SUMMARY_FILE"
fi

# Cleanup
rm -f "$TEMP_FILE" "$SUMMARY_FILE"

Добавить комментарий