import os
from glob import glob
from argparse import ArgumentParser
from copy import deepcopy
import logging
import json
import random
import pwd
import sh
__version__ = '1.1.0'
PBS_SERVER = 'p-shhq-hpc-pbs-m01'
QSUB = sh.Command('/opt/pbs/bin/qsub')
PBSNODES = sh.Command('/opt/pbs/bin/pbsnodes')
powerflow_run = '/home/hpcsw/PBSsubmit/run.powerflow2'
dis_run = '/home/hpcsw/PBSsubmit/run.dis2'
logger = logging.getLogger('pbs_sub')
# 一些公共函数
def free_cores(queue):
    """
    free cores in queue
    """
    count = 0
    logger.debug(f'getting free cores in {queue}')
    try:
        raw_nodedata = json.loads(PBSNODES('-aS', '-F', 'json').stdout)['nodes']
        raw_nodedata2 = json.loads(PBSNODES('-Saj', '-F', 'json').stdout)['nodes']
    except:
        logger.error('PBS error, can not get pbs info')
        exit(-1)
    for k, v in raw_nodedata.items():
        if v.get('queue') != queue:
            continue
        else:
            core_free = int(raw_nodedata2[k].get('ncpus f/t').split('/')[0])
            count += core_free
    logger.debug(f'number of free cores in {queue} is {count}')
    return count
# 公共命令行参数
parser = ArgumentParser(description=__doc__,
                        epilog='if you have any question or suggestion, please contact with thuhak.zhou@nio.com')
parser.add_argument('-n', '--name', help='job name')
parser.add_argument('-l', '--log_level', choices=['error', 'info', 'debug'], default='info', help='logging level')
parser.add_argument('-W', '--wait', metavar='JOB_ID', help='depend on which job')
software = parser.add_subparsers(dest='software', help='supported software')
# powerflow 软件的命令行参数
powerflow = software.add_parser('powerflow', help='powerflow')
powerflow.add_argument('-q', '--queue', choices=["cfd", "cfd2", "cfdbs"], required=True, help='queue of job')
powerflow.add_argument('-o', '--outdir', help='output directory, default is as same as jobfile')
powerflow.add_argument('-c', '--core', type=int, choices=[32, 64, 128, 256, 512, 768, 1024],
                       help='how many cpu cores you wanna use. default value is half of the free cores in your queue')
powerflow.add_argument('jobfile', help='job file')
powerflow.add_argument('-v', '--version', choices=["4.4d", "5.3b", "5.3c", "5.4b", "5.5a", "5.5c", "5.5c2", "6-2019"],
                       default="5.5b", help='version of powerflow software')
powerflow.add_argument('--nts', metavar='TIMESTEPS', type=int, help='num timesteps')
pow_group = powerflow.add_mutually_exclusive_group()
powerflow.add_argument('--seed_bondaries', action='store_true', help='set seed_bondaries')
pow_group.add_argument('-r', '--resume', metavar='RESUME_FILE', help='resume file')
pow_group.add_argument('-s', '--seed', metavar='SEED_FILE', help='seed file')
powerflow.add_argument('--mme', action='store_true', help='mme checkpoint at end')
powerflow.add_argument('--full', action='store_true', help='full checkpoint at end')
powerflow.add_argument('--pt', action='store_true', help='ptherm nprocs and ptherm max unmatched ratio')
powerflow.add_argument('--dis', choices=['only', 'full'],
                       help='run discretize, when you set only, just run dis job, if your set full, powerflow job will be run after dis job is finish')
powerflow.add_argument('--vr', metavar='LEVEL', type=int, help='suppress vr level for discretize job')
# 其他软件参数位置
other = software.add_parser('other')
...
if __name__ == '__main__':
  	# 公共的
    uid = os.getuid()
    user = pwd.getpwuid(uid)[0]
    email = user + '@nio.com'
    args = parser.parse_args()
    soft = args.software
    waitfor = args.wait
    log_level = getattr(logging, args.log_level.upper())
    logging.basicConfig(level=log_level, format='%(asctime)s [%(levelname)s]: %(message)s',
                        datefmt="%Y-%m-%d %H:%M:%S")
    for handler in logging.root.handlers:
        handler.addFilter(logging.Filter('pbs_sub'))
    base_args = ['-m', 'abe', '-M', email]
    if waitfor:
        if '.' not in waitfor:
            waitfor = waitfor + '.' + PBS_SERVER
        var_w = f'depend=afterok:{waitfor}'
        base_args.extend(['-W', var_w])
    # powerflow 处理参数的处理过程
    if soft == 'powerflow':
				...
    # 其他软件处理过程
    elif soft == 'other':
      	...

class MetaData(type):
    def __init__(cls, clsname, bases, clsdict):
        super().__init__(clsname, bases, clsdict)   # 可以操作cls
        print('meta class')
class A(metaclass=MetaData):
    data = 'A'
    def __new__(cls, *args, **kwargs):
        print('running new')
        return super().__new__(cls, *args, **kwargs) #返回实例对象
    def __init_subclass__(cls, *args, **kwargs):
        print('running init_subclass')               #可以操作cls
    def __init__(self, *args, **kwargs):
        print('running init')

class B(A):
  ...

class C(B):
  ...

class D(A):
  def __init_subclass__(cls, *args, **kwargs):
    print('rewrite')
    super().__init_subclass__(cls, *args, **kwargs)

class F(D):
  ...

class E:
  def __init_subclass__(cls, *args, **kwargs):
    print('other init subclass')
    super().__init_subclass__(cls, *args, **kwargs)

class G(A, E):
  ...

class H(E, A):
  ...

#!/usr/bin/env python3.6
from software import MainParser
if __name__ == '__main__':
    parser = MainParser()
    parser.run()

from .mainparser import MainParser
from .powerflow import PowerFlow

"""
main parser
"""
# author: thuhak.zhou@nio.com
from argparse import ArgumentParser
from weakref import WeakValueDictionary
import logging
from logging.handlers import RotatingFileHandler
import json
import os
from os.path import dirname, join, abspath, isfile, isdir
from string import ascii_letters, digits
import pwd
import time
import sh
__version__ = '2.1.0'
__author__ = 'thuhak.zhou@nio.com'
class classproperty:
    def __init__(self, func):
        self.func = func
    def __get__(self, instance, cls):
        if cls is None:
            return self
        else:
            value = self.func(cls)
            setattr(cls, self.func.__name__, value)
            return value
class MainParser:
    """
    provide common args for pbs.
    argument parser of subclasses will regist in _all_software variable
    the request will be route to the right software subclass
    """
    _all_software = WeakValueDictionary()
    logger = logging.getLogger('pbs_sub')
    PBS_SERVER = 'p-shhq-hpc-pbs-m01'
    QSUB = sh.Command('/opt/pbs/bin/qsub')
    script_base = join(dirname(abspath(__file__)), 'run_scripts')
    def __init__(self):
        self.base_args = []
        self.parser = ArgumentParser(description=f"This script is used for pbs job submission, version: {__version__}",
                                     epilog=f'if you have any question or suggestion, please contact with {__author__}')
        self.parser.add_argument('-n', '--name', help='job name')
        self.parser.add_argument('-l', '--log_level', choices=['error', 'info', 'debug'], default='info',
                                 help='logging level')
        self.parser.add_argument('-W', '--wait', metavar='JOB_ID', help='depend on which job')
        self.parser.add_argument('--free_cores', action='store_true', help='show free cpu cores by queue')
        software = self.parser.add_subparsers(dest='software', help='software list')
        for soft in self._all_software:
            cls = self._all_software[soft]
            parser = software.add_parser(cls.__software__, help=f'(script version {cls.__version__})')
            cls.add_parser(parser)
    @classmethod
    def add_parser(cls, parser: ArgumentParser):
        """
        abc interface, implement this method in subclass
        """
        raise NotImplementedError
    @classmethod
    def handle(cls, args, base_args) -> list:
        """
        abc interface, implement this method in subclass
        args: argument args
        base_args: all argument provided by main parser
        :return all job ids in list
        """
        raise NotImplementedError
    def __init_subclass__(cls, **kwargs):
        """
        regist subclass
        """
        super().__init_subclass__(**kwargs)
        if not getattr(cls, '__software__') or not getattr(cls, '__version__'):
            raise NotImplementedError('you need to set __software__ and __version__ attribute in your subclass')
        cls._all_software[cls.__software__] = cls
    @classproperty
    def default_run(cls):
        """
        default run script
        """
        run_script = join(cls.script_base, cls.__software__ + '.sh')
        if not isfile(run_script):
            cls.logger.error(f'you need to put the script {run_script} first')
            exit(1)
        return run_script
    @classproperty
    def pbs_nodes_data(cls):
        """
        get node data from pbs
        """
        cls.logger.debug('getting pbs node info')
        try:
            PBSNODES = sh.Command('/opt/pbs/bin/pbsnodes')
            raw_nodedata = json.loads(PBSNODES('-aS', '-F', 'json').stdout)['nodes']
            raw_nodedata2 = json.loads(PBSNODES('-Saj', '-F', 'json').stdout)['nodes']
            for k in raw_nodedata.keys():
                raw_nodedata[k].update(raw_nodedata2[k])
            return raw_nodedata
        except Exception as e:
            cls.logger.error(f'PBS error, can not get pbs node info, reason: {str(e)}')
            exit(-1)
    @classproperty
    def pbs_job_data(cls):
        """
        get pbs job data
        """
        cls.logger.debug('getting pbs job info')
        try:
            QSTAT = sh.Command('/opt/pbs/bin/qstat')
            raw_data = sh.grep(QSTAT('-f', '-F', 'json'), '-v', 'Submit_arguments').stdout
            job_data = json.loads(raw_data)['Jobs']
            return job_data
        except Exception as e:
            cls.logger.error(f'PBS error, can not get pbs job info, reason: {str(e)}')
            exit(-1)
    @classmethod
    def free_cores(cls, queue: str) -> int:
        """
        get free cores in queue
        """
        count = 0
        cls.logger.debug(f'getting free cores in {queue}')
        pbsdata = cls.pbs_nodes_data
        for node in pbsdata.values():
            if node.get('queue') == queue:
                core_free = int(node.get('ncpus f/t').split('/')[0])
                count += core_free
        cls.logger.debug(f'number of free cores in {queue} is {count}')
        return count
    @classmethod
    def all_free_cores(cls) -> dict:
        """
        get all free cores in all queues
        """
        from collections import defaultdict
        cls.logger.debug('getting free cores')
        pbsdata = cls.pbs_nodes_data
        ret = defaultdict(int)
        for node in pbsdata.values():
            queue = node.get('queue')
            if queue:
                ret[queue] += int(node.get('ncpus f/t').split('/')[0])
        return ret
    @classmethod
    def check_jobid(cls, jid: str) -> bool:
        """
        check job id in pbs or not
        """
        cls.logger.debug(f'checking job {jid}')
        return jid in cls.pbs_job_data
    @classmethod
    def get_jid_info(cls, jid: str) -> dict:
        cls.logger.debug(f'getting job information for {jid}')
        try:
            QSTAT = sh.Command('/opt/pbs/bin/qstat')
            raw_info = sh.grep(QSTAT('-f', '-F', 'json', jid), '-v', 'Submit_arguments').stdout
            job_info = json.loads(raw_info)['Jobs'][jid]
            return job_info
        except json.decoder.JSONDecodeError:
            cls.logger.error(f'job {jid} is not json format')
            cls.logger.debug(raw_info)
            exit(1)
        except Exception as e:
            cls.logger.error(f'unable to get information from pbs, reason: {str(e)}')
            exit(1)
    @staticmethod
    def replace_id(args: list, jobid: str) -> None:
        """this function will change the state of args"""
        try:
            w_i = args.index('-W')
            args[w_i + 1] = f'depend=afterok:{jobid}'
        except ValueError:
            args.extend(['-W', f'depend=afterok:{jobid}'])
    @staticmethod
    def fix_jobname(name: str) -> str:
        valid_str = ascii_letters + digits + '_'
        ret = name
        for c in name:
            if c not in valid_str:
                ret = ret.replace(c, '_')
        return ret
    @staticmethod
    def get_ncpu(queue: str) -> int:
        return 16 if queue == 'cfdbs' else 32
    def run(self):
        """
        program entry
        """
        uid = os.getuid()
        user = pwd.getpwuid(uid)[0]
        args = self.parser.parse_args()
        log_level = getattr(logging, args.log_level.upper())
        handlers = [logging.StreamHandler()]
        logdir = '/var/log/pbs_sub'
        if isdir(logdir):
            loghandler = RotatingFileHandler(join(logdir, user + '.log'), maxBytes=10*1024*1024, backupCount=3, encoding='utf-8')
            handlers.append(loghandler)
        logging.basicConfig(level=log_level, format='%(asctime)s [%(levelname)s]: %(message)s',
                            datefmt="%Y-%m-%d %H:%M:%S", handlers=handlers)
        for handler in logging.root.handlers:
            handler.addFilter(logging.Filter('pbs_sub'))
        if args.free_cores:
            free_cores = self.all_free_cores()
            ali = max([len(x) for x in free_cores.keys()]) + 3
            for q, c in free_cores.items():
                space = ' ' * (ali - len(q))
                print(f'{q}:{space}{c}')
            return
        software = args.software
        email = user + '@nio.com'
        waitfor = args.wait
        base_args = ['-m', 'abe', '-M', email]
        if waitfor:
            if '.' not in waitfor:
                waitfor = waitfor + '.' + self.PBS_SERVER
            if not self.check_jobid(waitfor):
                self.logger.debug(f'invalid job id {waitfor}')
                exit(1)
            var_w = f'depend=afterok:{waitfor}'
            base_args.extend(['-W', var_w])
        if software in self._all_software:
            jids = self._all_software[software].handle(args, base_args)
            time.sleep(1)
            for jid in jids:
                job_info = self.get_jid_info(jid)
                job_stat = job_info.get('job_state')
                info = f'job {jid} is in state {job_stat}'
                comment = job_info.get('comment')
                if comment:
                    info += f',comment: {comment}'
                self.logger.info(info)
        else:
            self.parser.print_help()

# author: thuhak.zhou@nio.com
import os
import random
from copy import deepcopy
from glob import glob
import re
import sh
from .mainparser import MainParser
class PowerFlow(MainParser):
    __software__ = 'powerflow'
    __version__ = '2.1.0'
    base_dir = '/home/hpcsw/EXA/powerflow'
    @classmethod
    def add_parser(cls, parser):
        parser.add_argument('-q', '--queue', choices=["cfd", "cfd2", "cfdbs"], default="cfdbs", help='queue of job')
        parser.add_argument('-c', '--core', type=int, choices=[32, 64, 128, 256, 512, 768, 1024],
                               help='how many cpu cores you wanna use. default value is half of the free cores in your queue')
        parser.add_argument('jobfile', help='job file')
        parser.add_argument('-v', '--version',
                               choices=["4.4d", "5.3b", "5.3c", "5.4b", "5.5a", "5.5b", "5.5c", "5.5c2", "6-2019"],
                               default="5.5b", help='version of powerflow')
        parser.add_argument('--nts', metavar='TIMESTEPS', type=int, help='num timesteps')
        parser.add_argument('--seed_bondaries', action='store_true', help='set seed_bondaries')
        pow_group = parser.add_mutually_exclusive_group()
        pow_group.add_argument('-r', '--resume', metavar='RESUME_FILE', help='resume file')
        pow_group.add_argument('-s', '--seed', metavar='SEED_FILE', help='seed file')
        parser.add_argument('--mme', action='store_true', help='mme checkpoint at end')
        parser.add_argument('--full', action='store_true', help='full checkpoint at end')
        parser.add_argument('--pt', action='store_true', help='enable powerTHERM')
        parser.add_argument('--dis', choices=['only', 'full'],
                               help='run discretize, when you set only, just run dis job, if your set full, powerflow job will be run after dis job is finish')
        parser.add_argument('--vr', metavar='LEVEL', type=int, help='suppress vr level for discretize job')
        parser.add_argument('--postacoustics', choices=['only', 'full'],
                               help='run powerflow post process, contact with fuchao.wang@nio.com to get more information')
    @classmethod
    def default_cores(cls, queue):
        total_cores = cls.free_cores(queue)
        for c in (2048, 1536, 1024, 960, 512):
            if total_cores >= c:
                return c // 2
        return 128
    @classmethod
    def handle(cls, args, base_args):
        jids = []
        ...
        return jids

def add_host(hostname, ipaddress, port=22, other_info=None):
    '''
    add host to jumpserver
    '''
    ret = {'result': True, 'retcode': 0, 'comment': ''}
    return ret
def get_host(hostname=None, ipaddress=None):
    '''
    get host from jumpserver
    '''
    ret = {'result': True, 'retcode': 0, 'comment': ''}
    return ret
def remove_host(hostname):
    '''
    remove host from jumpserver
    '''
    ret = {'result': True, 'retcode': 0, 'comment': ''}
    return ret
def reset_host(ipaddress, other_info=None):
    '''
    reset jumpserver information
    '''
    ret = {'result': True, 'retcode': 0, 'comment': ''}
    return ret

#!py
try:
    from redis_cache import *
    cache = SimpleCache(50000, expire=7200, namespace='jumpserver')
    HAS_CACHE = True
except:
    HAS_CACHE = False
def run():
    ret = {}
    ip = [ip for ip in __grains__['ipv4'] if ip.startswith("10.")][0]
    try:
        ret = cache.get_json(ip)
    except:
        try:
            import salt.runner
            runner = salt.runner.RunnerClient(__opts__)
            info = runner.cmd(fun='jumpserver.get_host', kwarg={'ipaddress': ip}, print_event=False)
            ret = info['data'][0]
            if HAS_CACHE:
                cache.store_json(ip, ret)
        except:
            pass
    return {'jumpserver': ret}

#!/usr/bin/env python
def _sshd_conf(conf_file):
    '''
    parse sshd config file
    '''
    sshdconf = {'enable_passwd': True, 'permit_root': True, 'port': 22}
    try:
        with open(conf_file, 'r') as f:
            for l in f:
                if l.startswith('Port'):
                    sshdconf['port'] = int(l.split()[1].lower())
                if l.startswith('PasswordAuthentication'):
                    sshdconf['enable_passwd'] = True if l.split()[1].lower() == 'yes' else False
                elif l.startswith('PermitRootLogin'):
                    sshdconf['permit_root'] = True if l.split()[1].lower() == 'yes' else False
                    break
    except:
        return None
    return sshdconf
def run():
    grains = {}
    sshd = _sshd_conf('/etc/ssh/sshd_config')
    if sshd:
        grains['sshd'] = sshd
    return grains

#!py
def run():
    jump = __pillar__.get('jumpserver', {})
    aws = __pillar__.get('aws', {})
    qcloud = __pillar__.get('qcloud', {})
    current_info = {}
    other_info = {}
    other_info['os'] = __grains__['os']
    other_info['os_version'] = __grains__['osrelease']
    other_info['os_arch'] = __grains__['osarch']
    ip = [ip for ip in __grains__['ipv4'] if ip.startswith('10.')][0]
    hostname = __grains__['id']
    port = __salt__['grains.get']('sshd:port', 22)
    if aws:
        ip = aws['PrivateIpAddress']
        hostname = aws['Tags']['Name']
        other_info['vendor'] = 'aws'
        other_info['model'] = aws['InstanceType']
        public_ip = aws.get('PublicIpAddress')
        if public_ip:
            other_info['public_ip'] = public_ip
        created_by = aws['Tags'].get('Creator')
        if created_by:
            other_info['created_by'] = created_by
    elif qcloud:
        ip = qcloud['PrivateIpAddresses'][0]
        hostname = qcloud['InstanceName']
        other_info['vendor'] = 'qcloud'
        public_ips = qcloud.get('PublicIpAddresses')
        if public_ips:
            other_info['public_ip'] = public_ips[0]
        created_by = qcloud['Tags'].get('Creator')
        if created_by:
            other_info['created_by'] = created_by
    current_info['ipaddress'] = ip
    if __salt__['config.get']('master') != __salt__['pillar.get']('salt_info:core_master'):
        current_info['_syndic_handle'] = True
    if not jump:
        current_info['hostname'] = hostname
        current_info['port'] = port
        current_info['other_info'] = other_info
        ret = {'jumpserver/add_host': {'event.send': [{'data': current_info}]}}
        return ret
    else:
        if hostname != jump.get('hostname'):
            other_info['hostname'] = hostname
        if port != jump.get('port'):
            other_info['port'] = port
        fix_info = {k:other_info[k] for k in other_info.keys() if other_info[k] != jump[k]}
        if fix_info:
            current_info['other_info'] = fix_info
            ret = {'jumpserver/reset_host': {'event.send': [{'data': current_info}]}}
            return ret

#!py
# author: thuhak.zhou@nio.com
from collections import Iterable
import boto3
import salt.config
from redis_cache import *
from ipaddress import ip_address, ip_network
saltconfig = salt.config.master_config('/etc/salt/master')
def _encode_all(items, ignore_type=(unicode,str)):
    if isinstance(items,dict):
        new_item = {}
        for k in items:
            new_item[k.encode('ascii')] = _encode_all(items[k])
        return new_item
    elif isinstance(items, Iterable) and not isinstance(items, ignore_type):
        new_item = []
        for x in items:
            new_item.append(_encode_all(x))
        return new_item
    elif isinstance(items,unicode):
        return items.encode('ascii')
    else:
        return items
def _tran_to_json(items):
    if isinstance(items,dict):
        new_item = {}
        for k,v in items.items():
            new_item[k] = _tran_to_json(v)
        return new_item
    elif isinstance(items, list):
        new_item = []
        for x in items:
            new_item.append(_tran_to_json(x))
        return new_item
    elif isinstance(items, (str, int, float, unicode)):
        return items
    else:
        return str(items)
def _filter_ip(ip_list):
    valid_network = saltconfig['ip-range']
    for raw_ip in ip_list:
        ip = ip_address(raw_ip.decode('ascii'))
        for raw_network in valid_network:
            network = ip_network(raw_network)
            if ip in network:
                return raw_ip
def run():
    """get aws attribute for ec2 instance
    :returns: aws attribute
    """
    try:
        cache = SimpleCache(50000, expire=3600, namespace='aws')
        HAS_CACHE = True
    except:
        HAS_CACHE = False
    ip = _filter_ip(__salt__["grains.get"]("ipv4"))
    if HAS_CACHE:
        try:
            result = cache.get_json(ip)
            return {'aws': result}
        except:
            pass
    filters = [{'Name': 'private-ip-address', 'Values': [ip]}]
    creds = _encode_all(saltconfig['aws']['creds'])
    result = {}
    for cred in creds:
        ec2 = boto3.client('ec2', region_name=cred['region_name'],aws_access_key_id=cred['aws_access_key_id'], aws_secret_access_key=cred['aws_secret_access_key'])
        try:
            result = ec2.describe_instances(Filters=filters)['Reservations'][0]['Instances'][0]
            Tags = {}
            for t in result.get('Tags'):
                Tags[t['Key']] = t['Value']
            result['Tags'] = Tags
            if HAS_CACHE:
                cache.store_json(ip, _tran_to_json(result))
            return {'aws': result}
        except Exception as e:
            continue
    return {'aws': result}

#!/usr/bin/env python
# author: thuhak.zhou@nio.com
'''
refresh redis cache for aws pillar
remove minions which can not be found by aws api
'''
from collections import Iterable
import logging
from logging.handlers import RotatingFileHandler
from ipaddress import ip_address, ip_network
from threading import Thread
from Queue import Queue
import boto3
import salt.config
import salt.runner
import salt.client
import salt.wheel
from redis_cache import *
logger = logging.getLogger('aws-cache')
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s [%(levelname)s]: %(message)s', "%Y-%m-%d %H:%M:%S")
loghandler1 = RotatingFileHandler('/var/log/aws/saltcron', maxBytes=10*1024*1024, backupCount=7)
loghandler2 = logging.StreamHandler()
loghandler1.setFormatter(formatter)
loghandler2.setFormatter(formatter)
logger.addHandler(loghandler1)
logger.addHandler(loghandler2)
saltconfig = salt.config.master_config('/etc/salt/master')
wheels = salt.wheel.WheelClient(saltconfig)
runner = salt.runner.RunnerClient(saltconfig)
caller = salt.client.Caller()
def _encode_all(items, ignore_type=(unicode,str)):
    if isinstance(items,dict):
        new_item = {}
        for k in items:
            new_item[k.encode('ascii')] = _encode_all(items[k])
        return new_item
    elif isinstance(items, Iterable) and not isinstance(items, ignore_type):
        new_item = []
        for x in items:
            new_item.append(_encode_all(x))
        return new_item
    elif isinstance(items,unicode):
        return items.encode('ascii')
    else:
        return items
def _tran_to_json(items):
    if isinstance(items,dict):
        new_item = {}
        for k,v in items.items():
            new_item[k] = _tran_to_json(v)
        return new_item
    elif isinstance(items, list):
        new_item = []
        for x in items:
            new_item.append(_tran_to_json(x))
        return new_item
    elif isinstance(items, (str, int, float, unicode)):
        return items
    else:
        return str(items)
def _check_ip(ip):
    try:
        ip_o = ip_address(ip.decode('ascii'))
    except:
        return False
    valid_network = saltconfig['ip-range']
    for raw_network in valid_network:
        network = ip_network(raw_network)
        if ip_o in network:
            return True
    return False
def all_instance():
    creds = _encode_all(saltconfig['aws']['creds'])
    for cred in creds:
        ec2 = boto3.client('ec2', region_name=cred['region_name'],aws_access_key_id=cred['aws_access_key_id'], aws_secret_access_key=cred['aws_secret_access_key'])
        result = ec2.describe_instances()['Reservations']
        for i in result:
            instances = i['Instances']
            for instance in instances:
                Tags = {}
                old_tag = instance.get('Tags')
                if not old_tag:
                    logger.warning('find ec2 instance without tag')
                    logger.debug(str(instance))
                else:
                    for t in old_tag:
                        Tags[t['Key']] = t['Value']
                instance['Tags'] = Tags
                yield _tran_to_json(instance)
def GetServerList():
    '''
    get dead minions
    '''
    minions = runner.cmd('manage.down', print_event=False)
    return minions
def DelServer(serName):
    '''
    delete server keys
    '''
    logger.info('removing minion {} in salt'.format(serName))
    wheels.cmd_async({'fun': 'key.delete', 'match': serName})
if __name__ == '__main__':
    cache = SimpleCache(50000, expire=3600, namespace='aws')
    queue = Queue()
    t = Thread(target=lambda q: q.put(GetServerList()), args=(queue,))
    t.start()
    all_ec2 = set()
    logger.info('start freshing cache for aws ec2')
    for instance in all_instance():
        try:
            interfaces = instance.get('NetworkInterfaces')
            if not interfaces:
                logger.debug('instance without interface')
                logger.debug(str(instance))
                continue
            ip = interfaces[0]['PrivateIpAddress']
            if not _check_ip(ip):
                logger.warning('{} is not valid private ip'.format(ip))
                continue
            name = instance['Tags'].get('Name')
            if not name:
                logger.warning('find ec2 {} without Name Tag'.format(ip))
            else:
                all_ec2.add(name)
            logger.debug('refresh cache for {}'.format(ip))
            cache.store_json(ip, instance)
        except Exception as e:
            logger.error('some error happen for instance {}'.format(instance))
            continue
    t.join()
    minions = queue.get()
    for minion in minions:
        if minion not in all_ec2:
            DelServer(minion)

#!/usr/bin/env python
__virtual__ = 'valuecheck'
def check(name, rule=None, ignorecase=False, dotall=False):
    '''
    check if value match rule, if not specify rule, check if value exists.
    '''
    import re
    ret = {'name': name, 'changes': {}, 'result': True, 'comment': 'value check pass'}
    if rule is None:
        if not name:
            ret['result'] = False
            ret['comment'] = 'value not match'
    else:
        iflag = re.IGNORECASE if ignorecase else 0
        dflag = re.DOTALL if dotall else 0
        flags = iflag | dflag
        if not re.match(rule, name, flags=flags):
            ret['result'] = False
            ret['comment'] = 'value not match'
    return ret

def wrap(name, changes=None, result=True, comment=''):
    if changes is None:
        changes = dict()
    ret = {'name': name,
           'changes': changes,
           'result': result,
           'comment': comment
            }
    return ret

#!/usr/bin/env python3
from pyquery import PyQuery as pq
from glob import glob
from urllib.parse import urljoin
import os
import requests
class NetworkError(Exception):
    pass
base = 'https://dl.bintray.com/vmware/'
basepath = '/repo/photon/'
repos = ['photon_release_1.0_x86_64/noarch/','photon_release_1.0_x86_64/x86_64/','photon_updates_1.0_x86_64/noarch/','photon_updates_1.0_x86_64/x86_64/','lightwave/x86_64/','photon_extras/noarch/','photon_extras/x86_64/']
def download(repo):
    curdir = os.path.join(basepath,repo)
    os.chdir(curdir)
    current_rpms = set(glob('*.rpm'))
    url = urljoin(base,repo)
    print(url)
    r = requests.get(url)
    if r.status_code != 200:
        raise NetworkError
    else:
        d = pq(r.content)
        target = set(d('a').text().split()) - current_rpms
        for rpm in target:
            if not rpm.endswith('.rpm'):
                continue
            else:
                t = urljoin(url,rpm)
                os.system('wget -c -t 0 {} -O {}'.format(t,rpm))
def main():
    for repo in repos:
        download(repo)
if __name__ == "__main__":
    main()

#!/usr/bin/env python3
import os.path
import weakref
from glob import glob
from pyquery import PyQuery as pq
from argparse import ArgumentParser
import xlwt
style = xlwt.XFStyle()
ali = xlwt.Alignment()
borders = xlwt.Borders()
borders.left = xlwt.Borders.THICK
borders.right = xlwt.Borders.THICK
borders.top = xlwt.Borders.THICK
borders.bottom = xlwt.Borders.THICK
ali.horz = xlwt.Alignment.HORZ_CENTER
ali.vert = xlwt.Alignment.VERT_CENTER
ali.wrap = xlwt.Alignment.WRAP_AT_RIGHT
style.alignment = ali
style.borders = borders
class Node:
    def __init__(self):
        self._parent = None
        self.height = 1
        self.level = 1
    @property
    def parent(self):
        return self._parent if self._parent is None else self._parent()
    @parent.setter
    def parent(self,node):
        self._parent = weakref.ref(node)
    def __repr__(self):
        return type(self) + 'height: %s' % self.height
    def show(self):
        pass
    def _grow(self, h):
        if h == 0:
            return
        else:
            p = self
            p.height += h
            if p.parent is not None:
                p = p.parent
                p._grow(h)
    def _dive(self, d):
        pass
class Branch(Node):
    def __init__(self, name):
        super(Branch, self).__init__()
        self.name = name
        self.children = []
    def __str__(self):
        s = 'Node:%s,height:%s' % (self.name, self.height)
        return s
    def show(self):
        print(self)
        for child in self.children:
            child.show()
    def add_node(self, node):
        width = len(self.children)
        self.children.append(node)
        node.parent = self
        node._dive(self.level)
        if width >= 1:
            self._grow(node.height)
        else:
            self._grow(node.height - 1)
    def _dive(self, d):
        self.level += d
        for child in self.children:
            child._dive(d)
class Leaf(Node):
    def __init__(self):
        super(Leaf, self).__init__()
        self.table = []
    def __str__(self):
        return str(self.height)
    def show(self):
        print(self)
    def add_data(self, data):
        width = len(data)
        if self.table == []:
            width -= 1
        self.table.extend(data)
        self._grow(width)
    def _dive(self, d):
        self.level += d
def walk(p, cur_node):
    for n in p.children():
        child = pq(n)
        if child.attr('class').startswith('he'):
            for g in child.children():
                grandchild = pq(g)
                if grandchild.attr('class') == 'sectionTitle':
                    node = Branch(child.text())
                    cur_node.add_node(node)
                    cur_node = node
                elif grandchild.is_('table'):
                    leaf = Leaf()
                    data = []
                    for t in grandchild('tr'):
                        td = pq(t).find('td')
                        if not td.attr('colspan') and td.eq(1).text() != '':
                            data.append((td.eq(0).text(), td.eq(1).text()))
                    cur_node.add_node(leaf)
                    leaf.add_data(data)
        elif child.attr('class') == 'container':
            walk(child, cur_node)
        else:
            continue
class GPOTree:
    def __init__(self, title, ctree, utree):
        self.title = title
        self.ctree = ctree
        self.utree = utree
    @classmethod
    def loadfile(cls, filename):
        gpo = cls.__new__(cls)
        with open(filename, 'rt', encoding='utf-16') as f:
            d = pq(''.join(f.readlines()))
        gpo.title = d('head').find('title').text().replace(' ', '_')[:31]
        p = g = d('div.gposummary')
        for i in range(10):
            p = p.next()
            if p.text() == '计算机配置(已启用)':
                c = p.next()
                C = Branch('计算机配置')
                walk(c, C)
                gpo.ctree = C
            elif p.text() == '用户配置(已启用)':
                u = p.next()
                U = Branch('用户配置')
                walk(u, U)
                gpo.utree = U
                break
        return gpo
    def export_to_excel_tree(self, filename):
        def traval(node, ws, x, y, style):
            if type(node) is Branch:
                ws.write_merge(x, x + node.height - 1, y, y, node.name, style)
                # print('write_merge(%s,%s,%s,%s,%s)'%(x,x+node.height-1,y,y,node.name))
                y += 1
                for child in node.children:
                    traval(child, ws, x, y, style)
                    x += child.height
            elif type(node) is Leaf:
                for i, entry in enumerate(node.table):
                    ws.write_merge(x + i, x + i, y, y, entry[0], style)
                    ws.write_merge(x + i, x + i, y + 1, y + 1, entry[1], style)
                    # print('write_merge(%s,%s,%s,%s,%s)'%(x+i,x+i,y,y,entry))
            else:
                return
        wb = xlwt.Workbook(encoding='utf-8')
        ws = wb.add_sheet(self.title)
        traval(self.ctree, ws, 0, 0, style)
        traval(self.utree, ws, self.ctree.height, self.ctree.height, style)
        wb.save(filename)
    def export_to_excel_table(self, filename):
        header_map = {1: 5, 2: 4, 3: 3, 4: 2, 5: 1}
        def traval(node, ws):
            nonlocal line
            if type(node) is Branch:
                ws.write_merge(line, line, 0, header_map.get(
                    node.level, 1), node.name, style)
                line += 2
                for child in node.children:
                    traval(child, ws)
            elif type(node) is Leaf:
                for i, entry in enumerate(node.table):
                    ws.write(line + i, 0, entry[0], style)
                    ws.write(line + i, 1, entry[1], style)
                line += node.height + 2
            else:
                return
        wb = xlwt.Workbook(encoding='utf-8')
        ws = wb.add_sheet(self.title)
        line = 0
        traval(self.ctree, ws)
        traval(self.utree, ws)
        wb.save(filename)
if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('-s', '--source', type=str,
                        required=True, help='组策略导出文件存储位置')
    parser.add_argument('-o', '--output', type=str,
                        required=True, help='组策略文件导出位置')
    args = parser.parse_args()
    source = args.source
    output = args.output
    if not os.path.isdir(source):
        raise SystemExit('source目录不存在')
    if not os.path.isdir(output):
        raise SystemExit('output目录不存在')
    gpo_files = glob(os.path.join(source, '*.html'))
    for f in gpo_files:
        gpo = GPOTree.loadfile(f)
        o = os.path.join(output, f.replace(
            source, output).replace('html', 'xls'))
        gpo.export_to_excel_table(o)