youget

#!/usr/bin/python

### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU Library General Public License as published by
### the Free Software Foundation; version 2 only
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU Library General Public License for more details.
###
### You should have received a copy of the GNU Library General Public License
### along with this program; if not, write to the Free Software
### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
### Copyright 2004-2006 Dag Wieers <dag@wieers.com>

### For SLES10 implementation, see:
###     http://lists.suse.com/archive/suse-sles-e/2006-Aug/0161.html

import fnmatch
import getopt
import getpass
import gzip
import os
import sys
import types
import urllib2
import urlparse

### Python 2.5 and higher
try:
    import xml.etree.ElementTree as ElementTree
except:
    ### Bummer, but cElementTree is considerably faster than ElementTree
    try:
        import cElementTree as ElementTree
    except:
        ### Then, hopefully this works ?
        try:
            import ElementTree
        except:
            ### No, it did not
            print >> sys.stderr, 'Error loading python module ElementTree, please install.'
            sys.exit(1)

__version__ = "$Revision: 4786 $"
# $Source$

VERSION = '0.8.4svn'


class Options:
    def __init__(self, args):
        self.cleanup = False
        self.credpath = None
        self.downloadall = False
        self.dryrun = False
        self.filter = None
        self.list = None
        self.quiet = False
        self.password = None
        self.username = None
        self.style = 'sles10'
        self.source = False
        self.verbose = 1

        try:
            opts, args = getopt.getopt(args,
                                       'd:hlnqp:s:u:v',
                                       ('credpath=', 'delete', 'download-all',
                                        'dryrun', 'filter=', 'help', 'list',
                                        'password=', 'quiet', 'source', 'style=',
                                        'username=', 'verbose', 'version'))
        except getopt.error, exc:
            print 'youget: %s, try youget -h for a list of all the options' % str(exc)
            sys.exit(1)

        for opt, arg in opts:
            if opt in ('-d', '--credpath'):
                self.credpath = arg
            elif opt in ('--delete', ):
                self.cleanup = True
            elif opt in ('--download-all', ):
                self.downloadall = True
            elif opt in ('--filter', ):
                self.filter = arg
                self.downloadall = True
            elif opt in ('-h', '--help'):
                self.usage()
                print
                self.help()
                sys.exit(0)
            elif opt in ('-l', '--list'):
                self.list = True
                self.downloadall = True
            elif opt in ('-n', '--dry-run'):
                self.dryrun = True
            elif opt in ['-p', '--password']:
                self.password = arg
            elif opt in ('-q', '--quiet'):
                self.quiet = True
            elif opt in ('--source'):
                self.source = True
            elif opt in ('-s', '--style'):
                self.style = arg
            elif opt in ['-u', '--username']:
                self.username = arg
            elif opt in ('-v', '--verbose'):
                self.verbose = self.verbose + 1
            elif opt in ('--version', ):
                self.version()
                sys.exit(0)

        if len(args) < 1:
            self.usage()
            print
            self.help()
            sys.exit(1)

        self.uri = args[0]

        if len(args) == 2:
            self.destination = args[1]
        else:
            self.destination = os.getcwd()

        if self.quiet:
            self.verbose = 0

        if self.verbose >= 3:
            print 'Verbosity set to level %d' % (self.verbose - 1)

    def version(self):
        print 'youget %s' % VERSION
        print 'Written by Dag Wieers <dag@wieers.com>'
        print
        print 'platform %s/%s' % (os.name, sys.platform)
        print 'python %s' % sys.version
        print
        print 'build revision $Rev: 4786 $'

    def usage(self):
        print 'usage: youget [options] URL'

    def help(self):
        print '''Download packages from Yast Online Update (YOU)

youget options:
  -d, --credpath=dir      credentials directory
      --delete            delete files that are not on the sender side
      --download-all      download all package versions available
      --filter            filter packages based on regexp
  -l, --list              list the available packages
  -n, --dry-run           show what would have been done
  -q, --quiet             minimal output
      --source            download source packages
  -v, --verbose           increase verbosity
  -vv, -vvv, -vvvv..      increase verbosity more
'''


class Set:
    def __init__(self):
        self.list = []

    def add(self, input):
        if input not in self.list:
            self.list.append(input)

    def delete(self, input):
        if input in self.list:
            self.list.removed(input)

    def difference(self, other):
        newlist = Set()
        for element in self.list:
            if element not in other.list:
                newlist.add(element)
        return newlist

    def sort(self):
        return self.list.sort()

    def __str__(self):
        return '\n\t' + '\n\t'.join([element[0] for element in self.list])

    def __len__(self):
        return len(self.list)


class MirrorException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)


def error(level, str):
    "Output error message"
    if level <= op.verbose:
        sys.stderr.write('youget: %s\n' % str)


def info(level, str):
    "Output info message"
    if level <= op.verbose:
        sys.stdout.write('%s\n' % str)


def die(ret, str):
    "Print error and exit with errorcode"
    error(0, str)
    sys.exit(ret)


def filelist(top):
    flist = []
    for root, dirs, files in os.walk(top):
        for file in files:
            flist.append(os.path.join(root, file))
    flist.sort()
    return flist


def remove(file):
    "Remove files or directories"
    if isinstance(file, types.StringType):
        if op.dryrun:
            return
        if os.path.islink(file):
            os.unlink(file)
        elif os.path.isdir(file):
            try:
                os.rmdir(file)
            except:
                os.path.walk(file, removedir, ())
                os.rmdir(file)
        elif os.path.isfile(file) or os.path.islink(file):
            os.unlink(file)
    else:
        for f in file:
            remove(f)


def mkdir(path):
    "Create a directory, and parents if needed"
    if op.dryrun:
        return
    if os.path.islink(path):
        os.unlink(path)
    if not os.path.exists(path):
        os.makedirs(path)


def httpget(opener, file, url, path=None):
    "Download file from url to local path"
    if path:
        mkdir(path)
    else:
        path = os.getcwd()
    if os.path.dirname(file):
        mkdir(os.path.join(path, os.path.dirname(file)))
    fdin = opener.open(os.path.join(url, file))
    fdout = open(os.path.join(path, file), 'w')
    fdout.write(fdin.read())
    fdin.close()
    fdout.close()


def mirroryou(url, path):
    'Check username/password and YOU mirror style'

    ### See if we find mcookie/partnernet in credpath
    if not op.username and not op.password:
        try:
            op.username = open(os.path.join(op.credpath, 'deviceid')).read().rstrip().rstrip('\0')
            op.password = open(os.path.join(op.credpath, 'secret')).read().rstrip().rstrip('\0')
            op.style = 'sles10'
        except:
            info(3, 'Credentials directory %s does not contain deviceid and secret files. (SLES10)' % op.credpath)

    ### See if we find mcookie/partnernet in credpath
    if not op.username and not op.password:
        try:
            op.username = open(os.path.join(op.credpath, 'mcookie')).read().rstrip().rstrip('\0')
            op.password = open(os.path.join(op.credpath, 'partnernet')).read().rstrip().rstrip('\0')
            op.style = 'nld9'
        except:
            info(3, 'Credentials directory %s does not contain mcookie and partnernet files. (NLD9)' % op.credpath)

    if op.credpath and not op.username and not op.password:
        die(2, 'No credentials found in %s.' % op.credpath)

    if not op.username:
        op.username = raw_input('YOU Username: ')

    if op.username and not op.password:
        op.password = getpass.getpass('YOU Password for user %s: ' % op.username)

    if op.style == 'sles10':
        mirroryou_sles10(url, path)
    elif op.style == 'nld9':
        mirroryou_nld9(url, path)
    else:
        mirroryou_sles10(url, path)


def mirroryou_sles10(url, path):
    'Mirror a channel from YOU (SLES10 style)'

    info(3, 'Using username %s with password %s.' % (op.username, op.password))

    ### Setting up connection
    host = urlparse.urlparse(url)[1]
    auth_handler = urllib2.HTTPDigestAuthHandler()
    auth_handler.add_password('Express', host, op.username, op.password)
    opener = urllib2.build_opener(auth_handler)
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]

    ### Download repodata for this channel
    info(2, 'Downloading packagelist and metadata from %s' % url)
    ### FIXME: check repomd.xml to see if any of the files have been updated
#    httpget(opener, 'repodata/repomd.xml.asc', url, path)
#    httpget(opener, 'repodata/repomd.xml.key', url, path)
#    httpget(opener, 'repodata/repomd.xml', url, path)
#    httpget(opener, 'repodata/filelists.xml.gz', url, path)
    httpget(opener, 'repodata/primary.xml.gz', url, path)
#    httpget(opener, 'repodata/patches.xml', url, path)

    ### Parse packagelist
    fd = gzip.open(os.path.join(path, 'repodata/primary.xml.gz'), 'r')
    tree = ElementTree.ElementTree(file=fd)
    root = tree.getroot()
    package_list = Set()
    for elem in root.getiterator('{http://linux.duke.edu/metadata/common}package'):
        pkgname = elem.find('{http://linux.duke.edu/metadata/common}location').get('href')
        pkgsize = int(elem.find('{http://linux.duke.edu/metadata/common}size').get('package'))
        package_list.add((pkgname, pkgsize))
    fd.close()
    package_list.sort()

    ### Download packages from the packagelist
    for filename, filesize in package_list.list:

        ### Filter packagelist
        if op.filter and not fnmatch.fnmatch(filename, op.filter):
            info(4, 'Packages %s excluded by filter' % filename)
            continue

        ### List only files if requested
        if op.list:
            info(0, filename)
            continue

        ### If file (or symlink target) exists
        if os.path.isfile(os.path.join(path, filename)):
            stat = os.stat(os.path.join(path, filename))
            if stat.st_size == filesize:
                info(3, 'File %s is already in %s' % (filename, path))
                continue
            else:
                info(2, 'File %s has wrong size (found: %s, expected: %s), refetching.' % (filename, stat.st_size, filesize))
                remove(os.path.join(path, filename))

        ### If symlink target does not exist, remove symlink
        elif os.path.islink(os.path.join(path, filename)):
            remove(os.path.join(path, filename))

        if op.dryrun:
            info(1, 'Not downloading package %s' % filename)
            continue

        info(2, 'Download %s (%s)' % (filename, filesize))
        httpget(opener, filename, url, path)

    ### Remove packages on the receiver side that are not on the sender side
    if op.cleanup:

        ### Collect receiver side
        receiver = Set()
        for file in filelist(os.path.join(path, 'rpm')):
            filename = file.split(path + '/')[1]
            filesize = os.stat(file).st_size
            receiver.add((filename, filesize))
        receiver.sort()

        ### Collect sender side
        sender = package_list

        ### Remove difference between receiver and sender
        cleanse = receiver.difference(sender)
        for filename, filesize in cleanse.list:
            info(3, 'Cleaning up obsolete file %s (%d kiB)' % (filename, filesize))
            remove(os.path.join(path, filename))


def mirroryou_nld9(url, path):
    'Mirror a channel from YOU (NLD9 style)'

    info(3, 'Using username %s with password %s.' % (op.username, op.password))

    ### Setting up connection
    host = urlparse.urlparse(url)[1]
    auth_handler = urllib2.HTTPDigestAuthHandler()
    auth_handler.add_password('Express', host, op.username, op.password)
    opener = urllib2.build_opener(auth_handler)
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]

    ### Download repodata for this channel
    info(2, 'Downloading packagelist from %s' % url)
    httpget(opener, 'packageinfo.xml.gz', url, path)

    ### Parse packagelist
    fd = gzip.open(os.path.join(path, 'packageinfo.xml.gz'), 'r')
    tree = ElementTree.ElementTree(file=fd)
    root = tree.getroot()
    package_list = Set()
    for elem in root.getiterator('package'):
        pkgname = elem.findtext('history/update/filename')
        pkgsize = int(elem.findtext('history/update/filesize'))
        package_list.add((pkgname, pkgsize))
    fd.close()
    package_list.sort()

    ### Download packages from the packagelist
    for filename, filesize in package_list.list:

        ### Filter packagelist
        if op.filter and not fnmatch.fnmatch(filename, op.filter):
            info(4, 'Packages %s excluded by filter' % filename)
            continue

        ### List only files if requested
        if op.list:
            info(0, filename)
            continue

        ### If file (or symlink target) exists
        if os.path.isfile(os.path.join(path, filename)):
            stat = os.stat(os.path.join(path, filename))
            if stat.st_size == filesize:
                info(3, 'File %s is already in %s' % (filename, path))
                continue
            else:
                info(2, 'File %s has wrong size (found: %s, expected: %s), refetching.' % (filename, stat.st_size, filesize))
                remove(os.path.join(path, filename))

        ### If symlink target does not exist, remove symlink
        elif os.path.islink(os.path.join(path, filename)):
            remove(os.path.join(path, filename))

        if op.dryrun:
            info(1, 'Not downloading package %s' % filename)
            continue

        info(2, 'Download %s (%s)' % (filename, filesize))
        httpget(opener, filename, url, path)

    ### Remove packages on the receiver side that are not on the sender side
    if op.cleanup:

        ### Collect receiver side
        receiver = Set()
        for file in filelist(os.path.join(path, 'rpm')):
            filename = file.split(path + '/')[1]
            filesize = os.stat(file).st_size
            receiver.add((filename, filesize))
        receiver.sort()

        ### Collect sender side
        sender = package_list

        ### Remove difference between receiver and sender
        cleanse = receiver.difference(sender)
        for filename, filesize in cleanse.list:
            info(3, 'Cleaning up obsolete file %s (%d kiB)' % (filename, filesize))
            remove(os.path.join(path, filename))


def main():
    try:
        mirroryou(op.uri, op.destination)
    except Exception, e:
        die(1, e)

### Unbuffered sys.stdout
sys.stdout = os.fdopen(1, 'w', 0)
sys.stderr = os.fdopen(2, 'w', 0)

### Workaround for python <= 2.2.1
try:
    True, False
except NameError:
    True = 1
    False = 0

### Main entrance
if __name__ == '__main__':
    exitcode = 0

    op = Options(sys.argv[1:])
    try:
        main()
    except KeyboardInterrupt, e:
        die(6, 'Exiting on user request')
    sys.exit(exitcode)

# vim:ts=4:sw=4:et