#!/usr/bin/python
#
# Copyright 2011 Dell, Inc.
#   by Matt Domsch <Matt_Domsch@dell.com>
# MIT/X11 license


__requires__ = 'TurboGears[future]'
import pkg_resources
pkg_resources.require("TurboGears")

import sys
import datetime as dt
import radix
from optparse import OptionParser
import dns.resolver
from IPy import IP
import textwrap
import logging
import email

from sqlobject import *
import turbogears

import smtplib
from fedora.client.fas2 import AccountSystem

sys.path.append('/usr/share/mirrormanager/server')
from mirrormanager.model import *
from mirrormanager.lib import manage_pidfile, remove_pidfile

from turbogears.database import PackageHub
from turbogears import config
hub = __connection__ = None


pidfile='/var/run/mirrormanager/errorcheck.pid'
internet2_netblocks_file = '/var/lib/mirrormanager/i2_netblocks.txt'
global_netblocks_file = '/var/lib/mirrormanager/global_netblocks.txt'
internet2_tree = None
global_tree = None
one_day = dt.timedelta(days=1)
logger = None
textwrapper = textwrap.TextWrapper(subsequent_indent = '  ')
options = None

class SiteResult(object):

    def __init__(self, site):
        self.site = site
        self.messages = []
        self.admin_emails = []

def send_mail(sender, to, subject, body, cc=[], bcc=[]):
    message = email.mime.text.MIMEText(body.encode('utf-8'), 'plain', 'utf-8')
    message['Subject'] = email.header.Header(subject, 'utf-8')
    message['From'] = email.header.Header(u'mirror-admin@fedoraproject.org', u'utf-8')
    message['To'] = email.header.Header(u', '.join(to), 'utf-8')
    message['Cc'] = email.header.Header(u', '.join(cc), 'utf-8')

    try:
        smtp = smtplib.SMTP('bastion', 25)
    
        smtp.sendmail(u'mirror-admin@fedoraproject.org',
                      to + cc + bcc,
                      message.as_string())
    except:
        logger.exception("Error sending email")
        logger.debug("Email message follows:")
        try:
            logger.debug(message.as_string())
        except:
            pass

    try:
        smtp.quit()
    except:
        pass


def name_to_ips(name):
    namestotry=[]
    result=set()
    records = []

    try:
        records = dns.resolver.query(name, 'CNAME')
    except:
        pass
    if len(records) > 0:
        for rdata in records:
            namestotry.append(rdata.to_text())
    else:
        namestotry = [name]

    records = []
    recordtypes=('A', 'AAAA')
    for name in namestotry:
        for r in recordtypes:
            try:
                records = dns.resolver.query(name, r)
            except:
                pass
            for rdata in records:
                try:
                    ip = IP(str(rdata.to_text()))
                except:
                    raise
                result.add(ip)
    return result

def convert_6to4_v4(ip):
    all_6to4 = ip('2002::/16')
    if ip.version() != 6 or ip not in all_6to4:
        return None
    parts=ip.strnormal().split(':')

    ab = int(parts[1],16)
    a = (ab >> 8) & 0xff
    b = ab & 0xff
    cd = int(parts[2],16)
    c = (cd >> 8) & 0xff
    d = cd & 0xff

    v4addr = '%d.%d.%d.%d' % (a,b,c,d)
    return ip(v4addr)

def convert_teredo_v4(ip):
    teredo_std = ip('2001::/32')
    teredo_xp  = ip('3ffe:831f::/32')
    if ip.version() != 6 or (ip not in teredo_std and ip not in teredo_xp):
        return None
    parts=ip.strnormal().split(':')

    ab = int(parts[6],16)
    a = ((ab >> 8) & 0xff) ^ 0xff
    b = (ab & 0xff) ^ 0xff
    cd = int(parts[7],16)
    c = ((cd >> 8) & 0xff) ^ 0xff
    d = (cd & 0xff) ^ 0xff

    v4addr = '%d.%d.%d.%d' % (a,b,c,d)
    return ip(v4addr)

gipv4 = None
gipv6 = None

def lookup_country(ip):
    clientcountry = None
    # attempt ipv6, then ipv6 6to4 as ipv4, then teredo, then ipv4
    try:
        if ip.version() == 6:
            if gipv6 is not None:
                clientcountry = gipv6.country_code_by_addr_v6(ip.strnormal())
            if clientcountry is None:
                # try the ipv6-to-ipv4 translation schemes
                for scheme in (convert_6to4_v4, convert_teredo_v4):
                    result = scheme(ip)
                    if result is not None:
                        ip = result
                        break
        if ip.version() == 4 and gipv4 is not None:
            clientcountry = gipv4.country_code_by_addr(ip.strnormal())
    except:
        pass

    return clientcountry


def open_geoip_databases():
    global gipv4
    global gipv6
    try:
        gipv4 = GeoIP.open("/usr/share/GeoIP/GeoIP.dat", GeoIP.GEOIP_STANDARD)
    except:
        gipv4=None
    try:
        gipv6 = GeoIP.open("/usr/share/GeoIP/GeoIPv6.dat", GeoIP.GEOIP_STANDARD)
    except:
        gipv6=None


def check_host(sr, host):
    if host.is_private() and not options.private:
        return

    if not host.is_active():
        return

    host_messages = []

    ips = name_to_ips(host.name)
    if len(ips) == 0 and not host.is_private():
        msg = u"* Host name %s not found in DNS.  Please use a FQDN for your host name.  You will also need to make the corresponding change in your report_mirror.conf file." % (host.name)
        host_messages.append(msg)
    else:
        if host.internet2:
            really_on_i2 = False
            for ip in ips:
                asn = lookup_ip_asn(internet2_tree, ip)
                if asn is not None:
                    really_on_i2 = True
                    break;
            if not really_on_i2:
                msg = u"* Host %s claims to be on Internet2, but is not in the Internet2 BGP table.  Please clear the Internet2 flag." % (host.name)
                host_messages.append(msg)
    
    if host.country is None:
        msg = u"* Host %s does not list the country it is in.  Please add the country." % (host.name)
        host_messages.append(msg)
    else:
        really_in_country = False
        country = None
        for ip in ips:
            country = lookup_country(ip)
            if country is not None and country == host.country:
                really_in_country = True
                break
        if country and not really_in_country:
            msg = u"* Host %s claims to be in country %s, but GeoIP reports them in %s.  Please contact mirror-admin@fedoraproject.org if you believe the GeoIP database is incorrect." % (host.name, host.country, country)
            host_messages.append(msg)


    if host.asn_clients:
        if host.asn is None:
            msg = u"* Host %s has set asn_clients, but has not set it's Autonomous System Number (ASN).  Please set a value for ASN (lookup tool at http://asn.cymru.com/)." % (host.name)
            host_messages.append(msg)
        else:
            for ip in ips:
                asn = lookup_ip_asn(global_tree, ip)
                if host.asn != asn:
                    msg = u"* Host %s has set ASN=%s, but appears in ASN %s in the global BGP table.  Please contact mirror-admin@fedoraproject.org if you believe your value is correct." % (host.name, host.asn, asn)
                    host_messages.append(msg)

    if len(host.categories) == 0:
        msg = u"* Host %s has no content Categories.  Please add Host Categories to the content your mirror carries." % (host.name)
        host_messages.append(msg)

    always_up2date = False

    for hc in host.categories:
        if len(hc.urls) == 0:
                msg = u"* Host %s Category %s has no URLs.  Please add a URL to the content your mirror carries, or if not carried, delete the Category." % (host.name, hc.category.name)
                host_messages.append(msg)            

        has_rsync = False
        for url in hc.urls:
            if url.url.startswith(u'rsync://'):
                has_rsync = True
                break
        if not has_rsync:
            if (host.internet2 and host.internet2_clients) or not host.is_private():
                msg = u"* Please consider adding an rsync URL for Category %s.  This will speed up the MirrorManager crawler and reduce load on your server.  URLs can be marked 'for other mirrors only', in which case they will not appear in publiclists, mirrorlists, or metalinks, but will be available to the MirrorManager crawler.  If necessary, you can restrict access to only allow from the MirrorManager crawler through a 'hosts allow = 209.132.181.0/24' line in your rsyncd.conf file." % hc.category.name
                host_messages.append(msg)

        if hc.always_up2date: always_up2date = True
        if len(hc.dirs) == 0 and not hc.always_up2date:
            msg = u"* Host %s Category %s has no up-to-date directories. "  % (host.name, hc.category.name)
            if len(hc.urls) > 0:
                msg += u"Check that your Category URL points to %s. " % (hc.category.topdir.name)

            msg += u"* Consult the crawler logs and check your report_mirror.conf."
            host_messages.append(msg)


    private_msg = u"  Private mirrors must run report_mirror after each rsync run completes in order for MirrorManager to know what content your mirror carries.  The crawler does not generally run against private mirrors."
    public_msg = u"  Public mirrors are encouraged to run report_mirror after each rsync run completes."
    if host.lastCheckedIn is None and not always_up2date:
        msg = u"* Host %s has never checked in with report_mirror." % (host.name)
        if host.is_private():
            msg += private_msg
        else:
            msg += public_msg
        host_messages.append(msg)
    elif host.lastCheckedIn is not None:
        if host.lastCheckedIn <  (dt.datetime.utcnow() - one_day):
            msg = u"* Host %s has not checked in with report_mirror in 24 hours, last check in %s." % (host.name, host.lastCheckedIn)
            if host.is_private():
                msg += private_msg
            else:
                msg += public_msg
            host_messages.append(msg)


    if host.is_private() and len(host.netblocks) == 0 and (host.asn is None or not host.asn_clients):
        msg = u"* Host %s is private has no netblocks or ASN set or asn_clients set.  Clients will not automatically be directed to your mirror.  Please set either netblocks or ASN value and asn_clients." % (host.name)
        host_messages.append(msg)

    if len(host_messages):
        host_mm_url = turbogears.url(u'/host/%d' % host.id)
        msg = u"* Details for Host name %s [%s]:" % (host.name, host_mm_url)
        sr.messages.append(msg)
        for msg in host_messages:
            sr.messages.append(u"    " + msg)

AS = None

def users_accountinfo(userlist):
    global AS
    result = {}
    if AS is None:
        if turbogears.config.get('identity.provider') == 'jsonfas2':
            from fedora.client.fas2 import AccountSystem
            AS = AccountSystem(username=options.fasuser, password=options.faspassword, cache_session=True)
    for user in userlist:
        person = AS.person_by_username(user)
        if person:
            result[user] = person
    return result

def site_admins(site):
    return [sa.username for sa in site.admins]

def lookup_ip_asn(tree, ip):
    """ @t is a radix tree
        @ip is an IPy.IP object which may be contained in an entry in l
        """
    node = tree.search_best(str(ip))
    if node is None:
        return None
    return node.data['asn']

def setup_netblocks(netblocks_file):
    tree = radix.Radix()
    if netblocks_file is not None:
        try:
            f = open(netblocks_file, 'r')
        except:
            return tree
        for l in f:
            try:
                s = l.split()
                start, mask = s[0].split('/')
                mask = int(mask)
                if mask == 0: continue
                asn = int(s[1])
                node = tree.add(s[0])
                node.data['asn'] = asn
            except:
                pass
        f.close()

    return tree

def check_site(site):
    if site.private and not options.private:
        return None
    if not site.admin_active or not site.user_active:
        return None

    sr = SiteResult(site)
    sr.admin_emails = []
    for username, person in users_accountinfo(site_admins(site)).iteritems():
        sr.admin_emails.append(person['email'])
        if person['status'] == u'inactive':
            msg = u"* Site Admin Username %s is inactive in FAS.  Visit https://admin.fedoraproject.org/accounts/ to log in and change your password, so you may make changes to your site's information in the MirrorManager database." % (username)
            sr.messages.append(msg)


    if len(site.hosts) == 0:
        msg = u"* Site %s has no Hosts.  Please add one or more Hosts to your Site entry.  Without Hosts, no clients can connect to your server." % (site.name)
        sr.messages.append(msg)

    for host in site.hosts:
        check_host(sr, host)
    return sr

def send_site_emails(sr, cc=[], sender=None):
    if len(sr.messages) == 0: return
    if len(sr.admin_emails) == 0: return
#    if not config.get('errorcheck.send_email', False): return
    projectname = config.get('mirrormanager.project.name', u'Fedora')
#    sender = config.get('errorcheck.mail_from')
#    if sender is None:
#        print "unable to send mail: mirrormanager.mail_from is not set"
#    return
    sender = u'mirror-admin@fedoraproject.org'
    subject = u'%s MirrorManager report for %s' % (projectname, sr.site.name)
    baseurl = turbogears.url(u'/')
    siteurl = turbogears.url(u'/site/%d' % (sr.site.id)
    body = u'''\
This is an automated report from the %(project)s MirrorManager at %(baseurl)s.
Site: %(site)s    %(siteurl)s

''' % dict(project=projectname, site=sr.site.name, siteurl=siteurl, baseurl=baseurl)

    for msg in sr.messages:
        body += textwrapper.fill(msg)
        body += u'\n'

    msg = u'''
If you no longer mirror Fedora, we thank you for your past support,
and ask that you please delete any references to your mirror from
the MirrorManager database. This will prevent any further such reports
from being mailed to you unnecessarily.

If you have any other questions, consult https://fedoraproject.org/wiki/Infrastructure/Mirroring
or mail to mirror-admin@fedoraproject.org.
'''
    msg = textwrapper.fill(msg)
    body += u'\n'
    body += msg
    body += u'\n'

    to = sr.admin_emails
    send_mail(sender, to, subject, body, cc=cc)


def check_objects():
    for p in Product.select():
        if not p.name:
            print "Error: Product %s has no name" % p

    for a in Arch.select():
        if not a.name:
            print "Error: Arch %s has no name" % a

    for c in Category.select():
        if not c.name:
            print "Error: Category %s has no name" % c
        if not c.product:
            print "Error: Category %s has no product" % c
        if not c.topdir:
            print "Error: Category %s has no topdir" % c

    for v in Version.select():
        if not v.name:
            print "Error: Version %s has no name" % v
        if not v.product:
            print "Error: Version %s has no product" % v
            
    for r in Repository.select():
        if not r.category:
            print "Error: Repository %s has no category" % r
        if not r.directory:
            print "Error: Repository %s has no directory" % r


def doit():
    global internet2_tree
    global global_tree
    internet2_tree = setup_netblocks(internet2_netblocks_file)
    global_tree    = setup_netblocks(global_netblocks_file)

    check_objects()

    if len(options.site):
        sites = [Site.get(id) for id in options.site]
    else:
        sites = Site.select().orderBy('id')

    for site in sites:
        try:
            if site.id < int(options.minsite): continue
            sr = check_site(site)
            if sr is not None:
                if len(sr.messages) > 0:
                    logger.info("==== Site %d: %s  ====\n" % (site.id, site.name))
                    for msg in sr.messages:
                        logger.info("\t%s" %  msg)
                if options.email:
                    send_site_emails(sr, options.cc)
        except KeyboardInterrupt:
            break
        except:
            logger.exception('')


def main():
    global options
    parser = OptionParser(usage=sys.argv[0] + " [options]")
    parser.add_option("-c", "--config",
                      dest="config", default='dev.cfg',
                      help="TurboGears config file to use")
    parser.add_option("--fasuser",
                      dest="fasuser", default=None,
                      help="Fedora Account System username for queries")
    parser.add_option("--faspassword",
                      dest="faspassword", default=None,
                      help="Fedora Account System password for queries")
    parser.add_option("--private",
                      dest="private", action="store_true", default=None,
                      help="include private mirrors")
    parser.add_option("--logfile",
                      dest="logfile", default='/var/log/mirrormanager/errorcheck.log',
                      help="FILE to write to")
    parser.add_option("-e", "--email",
                      dest="email", action="store_true", default=False,
                      help="Send emails to Site admins")
    parser.add_option("--cc", 
                      dest="cc", default=[], action="append",
                      help="Email address to include on cc:")
    parser.add_option("--site", 
                      dest="site", default=[], action="append",
                      help="Site(s) to scan, rather than all.  Can be used multiple times.")
    parser.add_option("--minsite", 
                      dest="minsite", default=0, action="store", metavar="ID",
                      help="Scan sites greater than ID.")
                      

    (options, args) = parser.parse_args()

    turbogears.update_config(configfile=options.config,
                             modulename="mirrormanager.config")


    

    if manage_pidfile(pidfile):
        print "another instance is running, try again later."
        sys.exit(1)

    global logger
    fmt = '%(asctime)s %(message)s'
    datefmt = '%m/%d/%Y %I:%M:%S %p'
    formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)
    logger = logging.getLogger('errorcheck')
    handler = logging.handlers.WatchedFileHandler(options.logfile, "a+b")
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    logger.setLevel(logging.DEBUG)

    global hub
    global __connection__
    hub = PackageHub("mirrormanager")
    __connection__ = hub
    
    doit()

    remove_pidfile(pidfile)


if __name__ == "__main__":
    sys.exit(main())
        
