pyqtgraph/tools/setupHelpers.py

# -*- coding: utf-8 -*-
import os, sys, re
try:
    from subprocess import check_output, check_call
except ImportError:
    import subprocess as sp
    def check_output(*args, **kwds):
        kwds['stdout'] = sp.PIPE
        proc = sp.Popen(*args, **kwds)
        output = proc.stdout.read()
        proc.wait()
        if proc.returncode != 0:
            ex = Exception("Process had nonzero return value %d" % proc.returncode)
            ex.returncode = proc.returncode
            ex.output = output
            raise ex
        return output

# Maximum allowed repository size difference (in kB) following merge.
# This is used to prevent large files from being inappropriately added to 
# the repository history.
MERGE_SIZE_LIMIT = 100

# Paths that are checked for style by flake and flake_diff
FLAKE_CHECK_PATHS = ['pyqtgraph', 'examples', 'tools']

# Flake style checks -- mandatory, recommended, optional
# See: http://pep8.readthedocs.org/en/1.4.6/intro.html
# and  https://flake8.readthedocs.org/en/2.0/warnings.html
FLAKE_MANDATORY = set([
    'E101',  #  indentation contains mixed spaces and tabs
    'E112',  #  expected an indented block
    'E122',  #  continuation line missing indentation or outdented
    'E125',  #  continuation line does not distinguish itself from next line
    'E133',  #  closing bracket is missing indentation

    'E223',  #  tab before operator
    'E224',  #  tab after operator
    'E242',  #  tab after ‘,’
    'E273',  #  tab after keyword
    'E274',  #  tab before keyword

    'E901',  #  SyntaxError or IndentationError
    'E902',  #  IOError
        
    'W191',  #  indentation contains tabs
        
    'W601',  #  .has_key() is deprecated, use ‘in’
    'W602',  #  deprecated form of raising exception
    'W603',  #  ‘<>’ is deprecated, use ‘!=’
    'W604',  #  backticks are deprecated, use ‘repr()’    
    ])

FLAKE_RECOMMENDED = set([
    'E124',  #  closing bracket does not match visual indentation
    'E231',  #  missing whitespace after ‘,’
    
    'E211',  #  whitespace before ‘(‘
    'E261',  #  at least two spaces before inline comment
    'E271',  #  multiple spaces after keyword
    'E272',  #  multiple spaces before keyword
    'E304',  #  blank lines found after function decorator

    'F401',  #  module imported but unused
    'F402',  #  import module from line N shadowed by loop variable
    'F403',  #  ‘from module import *’ used; unable to detect undefined names
    'F404',  #  future import(s) name after other statements
        
    'E501',  #  line too long (82 > 79 characters)
    'E502',  #  the backslash is redundant between brackets
    
    'E702',  #  multiple statements on one line (semicolon)
    'E703',  #  statement ends with a semicolon
    'E711',  #  comparison to None should be ‘if cond is None:’
    'E712',  #  comparison to True should be ‘if cond is True:’ or ‘if cond:’
    'E721',  #  do not compare types, use ‘isinstance()’

    'F811',  #  redefinition of unused name from line N
    'F812',  #  list comprehension redefines name from line N
    'F821',  #  undefined name name
    'F822',  #  undefined name name in __all__
    'F823',  #  local variable name ... referenced before assignment
    'F831',  #  duplicate argument name in function definition
    'F841',  #  local variable name is assigned to but never used
    
    'W292',  #  no newline at end of file

    ])

FLAKE_OPTIONAL = set([
    'E121',  #  continuation line indentation is not a multiple of four
    'E123',  #  closing bracket does not match indentation of opening bracket
    'E126',  #  continuation line over-indented for hanging indent
    'E127',  #  continuation line over-indented for visual indent
    'E128',  #  continuation line under-indented for visual indent
        
    'E201',  #  whitespace after ‘(‘
    'E202',  #  whitespace before ‘)’
    'E203',  #  whitespace before ‘:’
    'E221',  #  multiple spaces before operator
    'E222',  #  multiple spaces after operator
    'E225',  #  missing whitespace around operator
    'E227',  #  missing whitespace around bitwise or shift operator
    'E226',  #  missing whitespace around arithmetic operator
    'E228',  #  missing whitespace around modulo operator
    'E241',  #  multiple spaces after ‘,’
    'E251',  #  unexpected spaces around keyword / parameter equals
    'E262',  #  inline comment should start with ‘# ‘     
        
    'E301',  #  expected 1 blank line, found 0
    'E302',  #  expected 2 blank lines, found 0
    'E303',  #  too many blank lines (3)
        
    'E401',  #  multiple imports on one line

    'E701',  #  multiple statements on one line (colon)
        
    'W291',  #  trailing whitespace
    'W293',  #  blank line contains whitespace
        
    'W391',  #  blank line at end of file
    ])

FLAKE_IGNORE = set([
    # 111 and 113 are ignored because they appear to be broken.
    'E111',  #  indentation is not a multiple of four
    'E113',  #  unexpected indentation
    ])


#def checkStyle():
    #try:
        #out = check_output(['flake8', '--select=%s' % FLAKE_TESTS, '--statistics', 'pyqtgraph/'])
        #ret = 0
        #print("All style checks OK.")
    #except Exception as e:
        #out = e.output
        #ret = e.returncode
        #print(out.decode('utf-8'))
    #return ret


def checkStyle():
    """ Run flake8, checking only lines that are modified since the last
    git commit. """
    test = [ 1,2,3 ]
    
    # First check _all_ code against mandatory error codes
    print('flake8: check all code against mandatory error set...')
    errors = ','.join(FLAKE_MANDATORY)
    cmd = ['flake8', '--select=' + errors] + FLAKE_CHECK_PATHS
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    #ret = proc.wait()
    output = proc.stdout.read().decode('utf-8')
    ret = proc.wait()
    printFlakeOutput(output)
    
    # Check for DOS newlines
    print('check line endings in all files...')
    count = 0
    allowedEndings = set([None, '\n'])
    for path, dirs, files in os.walk('.'):
        for f in files:
            if os.path.splitext(f)[1] not in ('.py', '.rst'):
                continue
            filename = os.path.join(path, f)
            fh = open(filename, 'U')
            x = fh.readlines()
            endings = set(fh.newlines if isinstance(fh.newlines, tuple) else (fh.newlines,))
            endings -= allowedEndings
            if len(endings) > 0:
                print("\033[0;31m" + "File has invalid line endings: %s" % filename + "\033[0m")
                ret = ret | 2
            count += 1
    print('checked line endings in %d files' % count)
            
    
    # Next check new code with optional error codes
    print('flake8: check new code against recommended error set...')
    diff = subprocess.check_output(['git', 'diff'])
    proc = subprocess.Popen(['flake8', '--diff', #'--show-source',
                                '--ignore=' + errors],
                            stdin=subprocess.PIPE, 
                            stdout=subprocess.PIPE)
    proc.stdin.write(diff)
    proc.stdin.close()
    output = proc.stdout.read().decode('utf-8')
    ret |= printFlakeOutput(output)
    
    if ret == 0:
        print('style test passed.')
    else:
        print('style test failed: %d' % ret)
    return ret

def printFlakeOutput(text):
    """ Print flake output, colored by error category.
    Return 2 if there were any mandatory errors,
    1 if only recommended / optional errors, and
    0 if only optional errors.
    """
    ret = 0
    gotError = False
    for line in text.split('\n'):
        m = re.match(r'[^\:]+\:\d+\:\d+\: (\w+) .*', line)
        if m is None:
            print(line)
        else:
            gotError = True
            error = m.group(1)
            if error in FLAKE_MANDATORY:
                print("\033[0;31m" + line + "\033[0m")
                ret |= 2
            elif error in FLAKE_RECOMMENDED:
                print("\033[0;33m" + line + "\033[0m")
                #ret |= 1
            elif error in FLAKE_OPTIONAL:
                print("\033[0;32m" + line + "\033[0m")
            elif error in FLAKE_IGNORE:
                continue
            else:
                print("\033[0;36m" + line + "\033[0m")
    if not gotError:
        print("    [ no errors ]\n")
    return ret



def unitTests():
    """
    Run all unit tests (using py.test)
    Return the exit code.
    """
    try:
        if sys.version[0] == '3':
            out = check_output('PYTHONPATH=. py.test-3', shell=True)
        else:
            out = check_output('PYTHONPATH=. py.test', shell=True)
        ret = 0
    except Exception as e:
        out = e.output
        ret = e.returncode
    print(out.decode('utf-8'))
    return ret


def checkMergeSize(sourceBranch=None, targetBranch=None, sourceRepo=None, targetRepo=None):
    """
    Check that a git merge would not increase the repository size by MERGE_SIZE_LIMIT.
    """
    if sourceBranch is None:
        sourceBranch = getGitBranch()
        sourceRepo = '..'
        
    if targetBranch is None:
        if sourceBranch == 'develop':
            targetBranch = 'develop'
            targetRepo = 'https://github.com/pyqtgraph/pyqtgraph.git'
        else:
            targetBranch = 'develop'
            targetRepo = '..'
    
    workingDir = '__merge-test-clone'
    env = dict(TARGET_BRANCH=targetBranch, 
               SOURCE_BRANCH=sourceBranch, 
               TARGET_REPO=targetRepo, 
               SOURCE_REPO=sourceRepo,
               WORKING_DIR=workingDir,
               )
    
    print("Testing merge size difference:\n"
          "  SOURCE: {SOURCE_REPO} {SOURCE_BRANCH}\n"
          "  TARGET: {TARGET_BRANCH} {TARGET_REPO}".format(**env))
    
    setup = """
        mkdir {WORKING_DIR} && cd {WORKING_DIR} &&
        git init && git remote add -t {TARGET_BRANCH} target {TARGET_REPO} &&
        git fetch target {TARGET_BRANCH} && 
        git checkout -qf target/{TARGET_BRANCH} && 
        git gc -q --aggressive
        """.format(**env)
        
    checkSize = """
        cd {WORKING_DIR} && 
        du -s . | sed -e "s/\t.*//"
        """.format(**env)
    
    merge = """
        cd {WORKING_DIR} &&
        git pull -q {SOURCE_REPO} {SOURCE_BRANCH} && 
        git gc -q --aggressive
        """.format(**env)
    
    try:
        print("Check out target branch:\n" + setup)
        check_call(setup, shell=True)
        targetSize = int(check_output(checkSize, shell=True))
        print("TARGET SIZE: %d kB" % targetSize)
        print("Merge source branch:\n" + merge)
        check_call(merge, shell=True)
        mergeSize = int(check_output(checkSize, shell=True))
        print("MERGE SIZE: %d kB" % mergeSize)
        
        diff = mergeSize - targetSize
        if diff <= MERGE_SIZE_LIMIT:
            print("DIFFERENCE: %d kB  [OK]" % diff)
            return 0
        else:
            print("\033[0;31m" + "DIFFERENCE: %d kB  [exceeds %d kB]" % (diff, MERGE_SIZE_LIMIT) + "\033[0m")
            return 2
    finally:
        if os.path.isdir(workingDir):
            shutil.rmtree(workingDir)


def mergeTests():
    ret = checkMergeSize()
    ret |= unitTests()
    ret |= checkStyle()
    if ret == 0:
        print("\033[0;32m" + "\nAll merge tests passed." + "\033[0m")
    else:
        print("\033[0;31m" + "\nMerge tests failed." + "\033[0m")
    return ret


def listAllPackages(pkgroot):
    path = os.getcwd()
    n = len(path.split(os.path.sep))
    subdirs = [i[0].split(os.path.sep)[n:] for i in os.walk(os.path.join(path, pkgroot)) if '__init__.py' in i[2]]
    return ['.'.join(p) for p in subdirs]


def getInitVersion(pkgroot):
    """Return the version string defined in __init__.py"""
    path = os.getcwd()
    initfile = os.path.join(path, pkgroot, '__init__.py')
    init = open(initfile).read()
    m = re.search(r'__version__ = (\S+)\n', init)
    if m is None or len(m.groups()) != 1:
        raise Exception("Cannot determine __version__ from init file: '%s'!" % initfile)
    version = m.group(1).strip('\'\"')
    return version

def gitCommit(name):
    """Return the commit ID for the given name."""
    commit = check_output(['git', 'show', name], universal_newlines=True).split('\n')[0]
    assert commit[:7] == 'commit '
    return commit[7:]

def getGitVersion(tagPrefix):
    """Return a version string with information about this git checkout.
    If the checkout is an unmodified, tagged commit, then return the tag version.
    If this is not a tagged commit, return version-branch_name-commit_id.
    If this checkout has been modified, append "+" to the version.
    """
    path = os.getcwd()
    if not os.path.isdir(os.path.join(path, '.git')):
        return None
        
    # Find last tag matching "tagPrefix.*"
    tagNames = check_output(['git', 'tag'], universal_newlines=True).strip().split('\n')
    while True:
        if len(tagNames) == 0:
            raise Exception("Could not determine last tagged version.")
        lastTagName = tagNames.pop()
        if re.match(tagPrefix+r'\d+\.\d+.*', lastTagName):
            break
    gitVersion = lastTagName.replace(tagPrefix, '')
    
    # is this commit an unchanged checkout of the last tagged version? 
    lastTag = gitCommit(lastTagName)
    head = gitCommit('HEAD')
    if head != lastTag:
        branch = getGitBranch()
        gitVersion = gitVersion + "-%s-%s" % (branch, head[:10])
    
    # any uncommitted modifications?
    modified = False
    status = check_output(['git', 'status', '--porcelain'], universal_newlines=True).strip().split('\n')
    for line in status:
        if line != '' and line[:2] != '??':
            modified = True
            break        
                
    if modified:
        gitVersion = gitVersion + '+'

    return gitVersion

def getGitBranch():
    m = re.search(r'\* (.*)', check_output(['git', 'branch'], universal_newlines=True))
    if m is None:
        return ''
    else:
        return m.group(1)

def getVersionStrings(pkg):
    """
    Returns 4 version strings: 
    
    * the version string to use for this build,
    * version string requested with --force-version (or None)
    * version string that describes the current git checkout (or None).
    * version string in the pkg/__init__.py, 
    
    The first return value is (forceVersion or gitVersion or initVersion).
    """
    
    ## Determine current version string from __init__.py
    initVersion = getInitVersion(pkgroot='pyqtgraph')

    ## If this is a git checkout, try to generate a more descriptive version string
    try:
        gitVersion = getGitVersion(tagPrefix='pyqtgraph-')
    except:
        gitVersion = None
        sys.stderr.write("This appears to be a git checkout, but an error occurred "
                        "while attempting to determine a version string for the "
                        "current commit.\n")
        sys.excepthook(*sys.exc_info())

    # See whether a --force-version flag was given
    forcedVersion = None
    for i,arg in enumerate(sys.argv):
        if arg.startswith('--force-version'):
            if arg == '--force-version':
                forcedVersion = sys.argv[i+1]
                sys.argv.pop(i)
                sys.argv.pop(i)
            elif arg.startswith('--force-version='):
                forcedVersion = sys.argv[i].replace('--force-version=', '')
                sys.argv.pop(i)
                
                
    ## Finally decide on a version string to use:
    if forcedVersion is not None:
        version = forcedVersion
    elif gitVersion is not None and getGitBranch() != 'debian': # ignore git version if this is debian branch
        version = gitVersion
        sys.stderr.write("Detected git commit; will use version string: '%s'\n" % version)
    else:
        version = initVersion

    return version, forcedVersion, gitVersion, initVersion


from distutils.core import Command
import shutil, subprocess
from generateChangelog import generateDebianChangelog

class DebCommand(Command):
    description = "build .deb package using `debuild -us -uc`"
    maintainer = "Luke Campagnola <luke.campagnola@gmail.com>"
    debTemplate = "debian"
    debDir = "deb_build"
    
    user_options = []
    
    def initialize_options(self):
        self.cwd = None
        
    def finalize_options(self):
        self.cwd = os.getcwd()
        
    def run(self):
        version = self.distribution.get_version()
        pkgName = self.distribution.get_name()
        debName = "python-" + pkgName
        debDir = self.debDir
        
        assert os.getcwd() == self.cwd, 'Must be in package root: %s' % self.cwd
        
        if os.path.isdir(debDir):
            raise Exception('DEB build dir already exists: "%s"' % debDir)
        sdist = "dist/%s-%s.tar.gz" % (pkgName, version)
        if not os.path.isfile(sdist):
            raise Exception("No source distribution; run `setup.py sdist` first.")
        
        # copy sdist to build directory and extract
        os.mkdir(debDir)
        renamedSdist = '%s_%s.orig.tar.gz' % (debName, version)
        print("copy %s => %s" % (sdist, os.path.join(debDir, renamedSdist)))
        shutil.copy(sdist, os.path.join(debDir, renamedSdist))
        print("cd %s; tar -xzf %s" % (debDir, renamedSdist))
        if os.system("cd %s; tar -xzf %s" % (debDir, renamedSdist)) != 0:
            raise Exception("Error extracting source distribution.")
        buildDir = '%s/%s-%s' % (debDir, pkgName, version)
        
        # copy debian control structure
        print("copytree %s => %s" % (self.debTemplate, buildDir+'/debian'))
        shutil.copytree(self.debTemplate, buildDir+'/debian')
        
        # Write new changelog
        chlog = generateDebianChangelog(pkgName, 'CHANGELOG', version, self.maintainer)
        print("write changelog %s" % buildDir+'/debian/changelog')
        open(buildDir+'/debian/changelog', 'w').write(chlog)
        
        # build package
        print('cd %s; debuild -us -uc' % buildDir)
        if os.system('cd %s; debuild -us -uc' % buildDir) != 0:
            raise Exception("Error during debuild.")


class DebugCommand(Command):
    """Just for learning about distutils."""
    description = ""
    user_options = []
    def initialize_options(self):
        pass
    def finalize_options(self):
        pass
    def run(self):
        global cmd
        cmd = self
        print(self.distribution.name)
        print(self.distribution.version)


class TestCommand(Command):
    description = "Run all package tests and exit immediately with informative return code."
    user_options = []
    
    def run(self):
        sys.exit(unitTests())
        
    def initialize_options(self):
        pass
    
    def finalize_options(self):
        pass
    

class StyleCommand(Command):
    description = "Check all code for style, exit immediately with informative return code."
    user_options = []
    
    def run(self):
        sys.exit(checkStyle())
        
    def initialize_options(self):
        pass
    
    def finalize_options(self):
        pass

    
class MergeTestCommand(Command):
    description = "Run all tests needed to determine whether the current code is suitable for merge."
    user_options = []
    
    def run(self):
        sys.exit(mergeTests())
        
    def initialize_options(self):
        pass
    
    def finalize_options(self):
        pass