#!/bin/bash -e

# A complex script that creates a repository of deltas, that can be
# used by debdelta-upgrade for upgrading packages in.
# See also --help below.

# Copyright (C) 2006-10 Andrea Mennucci.
# License: GNU Library General Public License, version 2 or later


########### customize under here  vvvvvvvvvvvvv

#where is the full mirror of debian
debmir=/mirror/debian

#where to mirror from
host=ftp.debian.org

#where to store deltas
deltamir=/mirror/debian-deltas
#delete old debs from trash if the space here drops below kB :
minfreespace=2000000
#delete deltas and old_debs that are older than days (regardless of space)
old_delete_days=60

#where to keep snapshots of the older indexes of the archives (for the --snapshot option)
dists_snapshots=/mirror/debian-snapshots

#where is the debdeltas program
debdeltas=/usr/bin/debdeltas
#options to your taste
debdelta_opt="-v -d --test --delta-algo xdelta3 --gpg-home ~/debdelta/gnupg --sign 0xXXXXXXX "

##if you apply the 'trash patch' in contrib to debmirror, then you may use this:
debmirror_opt="--trash $deltamir/old_debs --method=http"
trash=$deltamir/old_debs
debmirror=/usr/local/bin/debmirror_2.4_mine

##otherwise this should work as well
#debmirror_opt='--method=http'
#trash=''
#debmirror=/usr/bin/debmirror

#note: files in $trash will be deleted by this script! use a dedicated dir!

#this should be in the same partition as the debian mirror $debmir
# (must be, if the trash is not used)
TMPDIR=/mirror/tmp
export TMPDIR


ARCHc='i386,amd64'
ARCHs='i386 amd64'

DISTc='lenny,squeeze,sid,experimental'
DISTs='lenny squeeze sid experimental'

########### customize over here ^^^^^^^^^^^^^^



DEBUG=''
VERBOSE=''
RM='rm'
DO_MIRROR=true
DO_CLEAN_DELTAS=''
DO_SNAPSHOT=''
while [ "$1" ] ; do
 case "$1" in
  -h|--help)
  cat <<EOF
Usage: debmirror-deltas [options]

This script (that contains many variables that must be customized!)
keeps a full mirror of the Debian repository (using 'debmirror'),
and it uses it to build a repository of deltas (using 'debdeltas').

Note that this script assumes that the current Debian
distributions are $DISTs .
It will generate deltas for the upgrades in those,
on arches $ARCHs .

Options:

-v   verbose  (passed to debmirror and debdeltas)
-d   debug    (passed to debmirror and debdeltas),
              also, do not 'rm' stuff, just tell
--no-mirror  if it crashed, resume the unfinished jobs
--clean-deltas , clean useless deltas (see debdeltas man page)
--snapshot     capture a snapshot of the archive (use daily)
EOF
  exit ;;
  -v)  VERBOSE="$VERBOSE -v"  ;;
  -d)  DEBUG=--debug ;  RM='echo -- rm ' ;;
  #used to finish interrupted runs
  --no-mirror) DO_MIRROR='' ;;
  --clean-deltas) DO_CLEAN_DELTAS=true ;;
  --snapshot) DO_SNAPSHOT=true ;;
  *) echo "Unknown option $1 , try --help" ;  exit 1 ;;
 esac
 shift
done


###################### some useful variables

#the lock used by debmirror
debmirlock=$debmir/Archive-Update-in-Progress-`hostname  -f`

b=`basename $0`

lockfile -r 10  /tmp/$b.lock || exit 1
trap "rm $VERBOSE -f /tmp/$b.lock" 0

olddists=""
log=/nonexistant

## profile debdeltas
#debdeltas="python -m cProfile -o /tmp/debdelta-profile-$(dirname $p | tr / _ ) $debdeltas"

today=`date +'%F'`
yyyymm=`date +'%Y-%m'`
test -d $deltamir/log/$yyyymm || mkdir $deltamir/log/$yyyymm

################## routines

freetrash () {
 test "$trash" || return
 age=${old_delete_days}
 blocksize=`stat -f  -c "%s" "$trash"`
 newsize=0 #trick to run at least once
 while test $age -ge 1 -a $newsize -le $minfreespace ; do
  find "$trash"  -name '*.deb' -type f -mtime +$age  |  xargs -r $RM $VERBOSE || true
  freeblocks=`stat -f -c "%a" "$trash"`
  newsize=`expr $freeblocks \* $blocksize / 1024`
  age=`expr $age - 5`
 done
}

cleanuppool () {
 find $deltamir/pool \
   \( -name '*debdelta-fails' -or -name '*debdelta-too-big' \
      -or -name  '*debdelta' \) -mtime +${old_delete_days} -type f |\
       xargs -r  $RM   $VERBOSE || true
 
 find $deltamir/pool/ -type d -empty | xargs -r rmdir $VERBOSE || true

 if test "$trash" ; then 
  find $trash -type f -not -name '*.deb' | xargs -r  $RM  $VERBOSE || true
 fi
}



run_debmirror () {
 if test -e $debmirlock ; then 
  echo Archive $debmir is locked 
  exit 1
 fi

 olddists=${TMPDIR:-/tmp}/olddists$$
 mkdir $olddists

 tm=$olddists/debmirror-stdout
 tme=$olddists/debmirror-stderr

 cp -a $debmir/dists $olddists

 ## if using trash, this is not needed : debdeltas will
 ##go fishing in the trash dir to find all deleted deltas
 if ! test "trash"  ; then
  cp -al $debmir/pool $olddists
 fi

 $debmirror  $debmir --nosource \
  $debmirror_opt  --arch=$ARCHc \
  --section=main,contrib,non-free \
  -v $DEBUG -h $host -d $DISTc \
     > $tm 2> $tme ; dme=$? 

 if test "$dme" != 0  ; then
   echo debmirror failed , exit code $dme , stdout  $tm 
   awk '{ print "> " $0 }' $tm 
   echo debmirror failed stderr $tme 
   awk '{ print "> " $0 }' $tme
   $RM $VERBOSE $tm $tme
   $RM -f $debmirlock || true
   #rm -r $debmir/.temp
   cleanuppool
   freetrash
   exit 1
 fi
 $RM $VERBOSE $tm $tme
}



findmelog () {
 date=`date +'%F-%H'`
 log=$deltamir/log/$yyyymm/${date}.log
 if test -r $log -o -r $log.gz ; then 
  for i in 0 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z ; do 
   if test -r $log -o -r $log.gz ; then 
     log=$deltamir/log/$yyyymm/${date}.$i.log
   fi
  done
 fi
}

created_deltas=""

run_debdeltas  () {
 if ! test "$olddists" -a -r "$olddists/dists" ; then
  echo "run_debdelta : bad $olddists"
  return
 fi
 findmelog
 (
  exec >> $log
  exec 2>> $log
  echo -n --------------oldnew pass----   ; date   ;
  echo "------ tmpdir was in $olddists"
  echo -n ---- options to debdelta ;  echo -- $debdelta_opt

  for dist in $DISTs ; do
   echo -------------- $dist ----
   #lenny contains debdelta 0.27, that does not understand lzma
   if test "$dist" = lenny ; then x="--disable-feature lzma" ; else x="" ; fi
   for p in $( cd $debmir ; find dists/$dist/ -name Packages.gz) ; do
    if test -r ${dists_snapshots}/before.4/$p ; then o="--old  ${dists_snapshots}/before.4/$p" ; else o='' ; fi
    echo ------ debdelta $x $o --old $olddists/$p $p
    $debdeltas $VERBOSE $debdelta_opt  \
       --alt $trash $x $o \
       --old $olddists/$p \
       --dir $deltamir//  $debmir/$p
   done
  done
 )

 #uncomment for added verbosity
 #egrep -3 -i 'error|warning' $log || true
 if grep -iq 'error' $log ; then
  #some error occurred
  test ! -r "$log".gz && { gzip -9 $log ; log=${log}.gz ; }
  echo "-----ERRORS--- full log is in $log , data in $olddists "
 else
  #no error occurred, clean up
  if test "$olddists" ; then
   if test -r "$olddists/dists" ; then
    $RM -r  "$olddists/dists"
   fi
   if test -r "$olddists/pool" ; then
    $RM -r  "$olddists/pool"
   fi
   #this is a bit exotic , but unfortunately 'test' does not test empty dirs
   if find "$olddists" -maxdepth 0 -type d -empty | grep -q "$olddists" ; then
    rmdir "$olddists" ; else echo Not empty "$olddists" ; 
   fi
  fi
  if grep -v '^---' $log | grep -v '^ Total running time' |  grep  -q '.' ; then
   test ! -r "$log".gz && gzip -9 $log
   created_deltas=1
  else
   rm $log
  fi
 fi
}


run_debdeltas_clean () {
   echo -------------- cleanup delta pool
   $debdeltas --clean-deltas -n 0 \
     --dir $deltamir// \
       $(  for arch in $ARCHs ; do
            for sec in main contrib non-free ; do
              for dist in $DISTs ; do
                echo $debmir/dists/$dist/$sec/binary-$arch/Packages.gz 
           done ; done ; done )
}


########################################### code


if test "${DO_CLEAN_DELTAS}" ; then
 run_debdeltas_clean
 cleanuppool
 freetrash
fi

if test "${DO_SNAPSHOT}" ; then
  if test -e $debmirlock ; then 
   echo Archive $debmir is locked, cannot --snapshot 
  elif test -w ${dists_snapshots} -a -d ${dists_snapshots} ; then
   test -e ${dists_snapshots}/before.4 && rm -r ${dists_snapshots}/before.4
   test -e ${dists_snapshots}/before.3 && mv ${dists_snapshots}/before.3 ${dists_snapshots}/before.4
   test -e ${dists_snapshots}/before.2 && mv ${dists_snapshots}/before.2 ${dists_snapshots}/before.3
   test -e ${dists_snapshots}/before.1 && mv ${dists_snapshots}/before.1 ${dists_snapshots}/before.2
   mkdir ${dists_snapshots}/before.1
   cp -a $debmir/dists ${dists_snapshots}/before.1/.
  else
   echo Please set dists_snapshots to a directory where I can write, not ${dists_snapshots} 1>&2
  fi
fi

if [ "${DO_MIRROR}" ] ; then
 run_debmirror
else
 echo ---------- skipped mirror step
fi

if [ "$olddists"  -a -r "$olddists" ] ; then 
 run_debdeltas
fi

for olddists  in ${TMPDIR:-/tmp}/olddists* ; do
 if [ "$olddists"  -a -r "$olddists" ] ; then
   echo ----------- there are old dists in $olddists
 fi
done

