diff options
Diffstat (limited to 'third_party/Python/module/pexpect-2.4/examples')
23 files changed, 4343 insertions, 0 deletions
diff --git a/third_party/Python/module/pexpect-2.4/examples/README b/third_party/Python/module/pexpect-2.4/examples/README new file mode 100644 index 000000000000..8f2581e05e7c --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/README @@ -0,0 +1,72 @@ +This directory contains scripts that give examples of using Pexpect. + +hive.py + This script creates SSH connections to a list of hosts that + you provide. Then you are given a command line prompt. Each + shell command that you enter is sent to all the hosts. The + response from each host is collected and printed. For example, + you could connect to a dozen different machines and reboot + them all at once. + +script.py + This implements a command similar to the classic BSD "script" command. + This will start a subshell and log all input and output to a file. + This demonstrates the interact() method of Pexpect. + +fix_cvs_files.py + This is for cleaning up binary files improperly added to + CVS. This script scans the given path to find binary files; + checks with CVS to see if the sticky options are set to -kb; + finally if sticky options are not -kb then uses 'cvs admin' + to set the -kb option. + +ftp.py + This demonstrates an FTP "bookmark". + This connects to an ftp site; does a few ftp commands; and then gives the user + interactive control over the session. In this case the "bookmark" is to a + directory on the OpenBSD ftp server. It puts you in the i386 packages + directory. You can easily modify this for other sites. + This demonstrates the interact() method of Pexpect. + +monitor.py + This runs a sequence of system status commands on a remote host using SSH. + It runs a simple system checks such as uptime and free to monitor + the state of the remote host. + +passmass.py + This will login to a list of hosts and change the password of the + given user. This demonstrates scripting logins; although, you could + more easily do this using the pxssh subclass of Pexpect. + See also the "hive.py" example script for a more general example + of scripting a collection of servers. + +python.py + This starts the python interpreter and prints the greeting message backwards. + It then gives the user interactive control of Python. It's pretty useless! + +rippy.py + This is a wizard for mencoder. It greatly simplifies the process of + ripping a DVD to mpeg4 format (XviD, DivX). It can transcode from any + video file to another. It has options for resampling the audio stream; + removing interlace artifacts, fitting to a target file size, etc. + There are lots of options, but the process is simple and easy to use. + +sshls.py + This lists a directory on a remote machine. + +ssh_tunnel.py + This starts an SSH tunnel to a remote machine. It monitors the connection + and restarts the tunnel if it goes down. + +uptime.py + This will run the uptime command and parse the output into python variables. + This demonstrates using a single regular expression to match the output + of a command and capturing different variable in match groups. + The regular expression takes into account a wide variety of different + formats for uptime output. + +df.py + This collects filesystem capacity info using the 'df' command. + Tuples of filesystem name and percentage are stored in a list. + A simple report is printed. Filesystems over 95% capacity are highlighted. + diff --git a/third_party/Python/module/pexpect-2.4/examples/astat.py b/third_party/Python/module/pexpect-2.4/examples/astat.py new file mode 100644 index 000000000000..82fa3c68b705 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/astat.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +"""This runs Apache Status on the remote host and returns the number of requests per second. + +./astat.py [-s server_hostname] [-u username] [-p password] + -s : hostname of the remote server to login to. + -u : username to user for login. + -p : Password to user for login. + +Example: + This will print information about the given host: + ./astat.py -s www.example.com -u mylogin -p mypassword + +""" + +import os, sys, time, re, getopt, getpass +import traceback +import pexpect, pxssh + +def exit_with_usage(): + + print globals()['__doc__'] + os._exit(1) + +def main(): + + ###################################################################### + ## Parse the options, arguments, get ready, etc. + ###################################################################### + try: + optlist, args = getopt.getopt(sys.argv[1:], 'h?s:u:p:', ['help','h','?']) + except Exception, e: + print str(e) + exit_with_usage() + options = dict(optlist) + if len(args) > 1: + exit_with_usage() + + if [elem for elem in options if elem in ['-h','--h','-?','--?','--help']]: + print "Help:" + exit_with_usage() + + if '-s' in options: + hostname = options['-s'] + else: + hostname = raw_input('hostname: ') + if '-u' in options: + username = options['-u'] + else: + username = raw_input('username: ') + if '-p' in options: + password = options['-p'] + else: + password = getpass.getpass('password: ') + + # + # Login via SSH + # + p = pxssh.pxssh() + p.login(hostname, username, password) + p.sendline('apachectl status') + p.expect('([0-9]+\.[0-9]+)\s*requests/sec') + requests_per_second = p.match.groups()[0] + p.logout() + print requests_per_second + +if __name__ == "__main__": + try: + main() + except Exception, e: + print str(e) + traceback.print_exc() + os._exit(1) + diff --git a/third_party/Python/module/pexpect-2.4/examples/bd_client.py b/third_party/Python/module/pexpect-2.4/examples/bd_client.py new file mode 100644 index 000000000000..564739a0aad5 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/bd_client.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +"""This is a very simple client for the backdoor daemon. This is intended more +for testing rather than normal use. See bd_serv.py """ + +import socket +import sys, time, select + +def recv_wrapper(s): + r,w,e = select.select([s.fileno()],[],[], 2) + if not r: + return '' + #cols = int(s.recv(4)) + #rows = int(s.recv(4)) + cols = 80 + rows = 24 + packet_size = cols * rows * 2 # double it for good measure + return s.recv(packet_size) + +#HOST = '' #'localhost' # The remote host +#PORT = 1664 # The same port as used by the server +s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +s.connect(sys.argv[1])#(HOST, PORT)) +time.sleep(1) +#s.setblocking(0) +#s.send('COMMAND' + '\x01' + sys.argv[1]) +s.send(':sendline ' + sys.argv[2]) +print recv_wrapper(s) +s.close() +sys.exit() +#while True: +# data = recv_wrapper(s) +# if data == '': +# break +# sys.stdout.write (data) +# sys.stdout.flush() +#s.close() + diff --git a/third_party/Python/module/pexpect-2.4/examples/bd_serv.py b/third_party/Python/module/pexpect-2.4/examples/bd_serv.py new file mode 100644 index 000000000000..b7def9e14022 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/bd_serv.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python + +"""Back door shell server + +This exposes an shell terminal on a socket. + + --hostname : sets the remote host name to open an ssh connection to. + --username : sets the user name to login with + --password : (optional) sets the password to login with + --port : set the local port for the server to listen on + --watch : show the virtual screen after each client request +""" + +# Having the password on the command line is not a good idea, but +# then this entire project is probably not the most security concious thing +# I've ever built. This should be considered an experimental tool -- at best. +import pxssh, pexpect, ANSI +import time, sys, os, getopt, getpass, traceback, threading, socket + +def exit_with_usage(exit_code=1): + + print globals()['__doc__'] + os._exit(exit_code) + +class roller (threading.Thread): + + """This runs a function in a loop in a thread.""" + + def __init__(self, interval, function, args=[], kwargs={}): + + """The interval parameter defines time between each call to the function. + """ + + threading.Thread.__init__(self) + self.interval = interval + self.function = function + self.args = args + self.kwargs = kwargs + self.finished = threading.Event() + + def cancel(self): + + """Stop the roller.""" + + self.finished.set() + + def run(self): + + while not self.finished.isSet(): + # self.finished.wait(self.interval) + self.function(*self.args, **self.kwargs) + +def endless_poll (child, prompt, screen, refresh_timeout=0.1): + + """This keeps the screen updated with the output of the child. This runs in + a separate thread. See roller(). """ + + #child.logfile_read = screen + try: + s = child.read_nonblocking(4000, 0.1) + screen.write(s) + except: + pass + #while True: + # #child.prompt (timeout=refresh_timeout) + # try: + # #child.read_nonblocking(1,timeout=refresh_timeout) + # child.read_nonblocking(4000, 0.1) + # except: + # pass + +def daemonize (stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): + + '''This forks the current process into a daemon. Almost none of this is + necessary (or advisable) if your daemon is being started by inetd. In that + case, stdin, stdout and stderr are all set up for you to refer to the + network connection, and the fork()s and session manipulation should not be + done (to avoid confusing inetd). Only the chdir() and umask() steps remain + as useful. + + References: + UNIX Programming FAQ + 1.7 How do I get my program to act like a daemon? + http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 + + Advanced Programming in the Unix Environment + W. Richard Stevens, 1992, Addison-Wesley, ISBN 0-201-56317-7. + + The stdin, stdout, and stderr arguments are file names that will be opened + and be used to replace the standard file descriptors in sys.stdin, + sys.stdout, and sys.stderr. These arguments are optional and default to + /dev/null. Note that stderr is opened unbuffered, so if it shares a file + with stdout then interleaved output may not appear in the order that you + expect. ''' + + # Do first fork. + try: + pid = os.fork() + if pid > 0: + sys.exit(0) # Exit first parent. + except OSError, e: + sys.stderr.write ("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror) ) + sys.exit(1) + + # Decouple from parent environment. + os.chdir("/") + os.umask(0) + os.setsid() + + # Do second fork. + try: + pid = os.fork() + if pid > 0: + sys.exit(0) # Exit second parent. + except OSError, e: + sys.stderr.write ("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror) ) + sys.exit(1) + + # Now I am a daemon! + + # Redirect standard file descriptors. + si = open(stdin, 'r') + so = open(stdout, 'a+') + se = open(stderr, 'a+', 0) + os.dup2(si.fileno(), sys.stdin.fileno()) + os.dup2(so.fileno(), sys.stdout.fileno()) + os.dup2(se.fileno(), sys.stderr.fileno()) + + # I now return as the daemon + return 0 + +def add_cursor_blink (response, row, col): + + i = (row-1) * 80 + col + return response[:i]+'<img src="http://www.noah.org/cursor.gif">'+response[i:] + +def main (): + + try: + optlist, args = getopt.getopt(sys.argv[1:], 'h?d', ['help','h','?', 'hostname=', 'username=', 'password=', 'port=', 'watch']) + except Exception, e: + print str(e) + exit_with_usage() + + command_line_options = dict(optlist) + options = dict(optlist) + # There are a million ways to cry for help. These are but a few of them. + if [elem for elem in command_line_options if elem in ['-h','--h','-?','--?','--help']]: + exit_with_usage(0) + + hostname = "127.0.0.1" + port = 1664 + username = os.getenv('USER') + password = "" + daemon_mode = False + if '-d' in options: + daemon_mode = True + if '--watch' in options: + watch_mode = True + else: + watch_mode = False + if '--hostname' in options: + hostname = options['--hostname'] + if '--port' in options: + port = int(options['--port']) + if '--username' in options: + username = options['--username'] + print "Login for %s@%s:%s" % (username, hostname, port) + if '--password' in options: + password = options['--password'] + else: + password = getpass.getpass('password: ') + + if daemon_mode: + print "daemonizing server" + daemonize() + #daemonize('/dev/null','/tmp/daemon.log','/tmp/daemon.log') + + sys.stdout.write ('server started with pid %d\n' % os.getpid() ) + + virtual_screen = ANSI.ANSI (24,80) + child = pxssh.pxssh() + child.login (hostname, username, password) + print 'created shell. command line prompt is', child.PROMPT + #child.sendline ('stty -echo') + #child.setecho(False) + virtual_screen.write (child.before) + virtual_screen.write (child.after) + + if os.path.exists("/tmp/mysock"): os.remove("/tmp/mysock") + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + localhost = '127.0.0.1' + s.bind('/tmp/mysock') + os.chmod('/tmp/mysock',0777) + print 'Listen' + s.listen(1) + print 'Accept' + #s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + #localhost = '127.0.0.1' + #s.bind((localhost, port)) + #print 'Listen' + #s.listen(1) + + r = roller (0.01, endless_poll, (child, child.PROMPT, virtual_screen)) + r.start() + print "screen poll updater started in background thread" + sys.stdout.flush() + + try: + while True: + conn, addr = s.accept() + print 'Connected by', addr + data = conn.recv(1024) + if data[0]!=':': + cmd = ':sendline' + arg = data.strip() + else: + request = data.split(' ', 1) + if len(request)>1: + cmd = request[0].strip() + arg = request[1].strip() + else: + cmd = request[0].strip() + if cmd == ':exit': + r.cancel() + break + elif cmd == ':sendline': + child.sendline (arg) + #child.prompt(timeout=2) + time.sleep(0.2) + shell_window = str(virtual_screen) + elif cmd == ':send' or cmd==':xsend': + if cmd==':xsend': + arg = arg.decode("hex") + child.send (arg) + time.sleep(0.2) + shell_window = str(virtual_screen) + elif cmd == ':cursor': + shell_window = '%x%x' % (virtual_screen.cur_r, virtual_screen.cur_c) + elif cmd == ':refresh': + shell_window = str(virtual_screen) + + response = [] + response.append (shell_window) + #response = add_cursor_blink (response, row, col) + sent = conn.send('\n'.join(response)) + if watch_mode: print '\n'.join(response) + if sent < len (response): + print "Sent is too short. Some data was cut off." + conn.close() + finally: + r.cancel() + print "cleaning up socket" + s.close() + if os.path.exists("/tmp/mysock"): os.remove("/tmp/mysock") + print "done!" + +def pretty_box (rows, cols, s): + + """This puts an ASCII text box around the given string, s. + """ + + top_bot = '+' + '-'*cols + '+\n' + return top_bot + '\n'.join(['|'+line+'|' for line in s.split('\n')]) + '\n' + top_bot + +def error_response (msg): + + response = [] + response.append ("""All commands start with : +:{REQUEST} {ARGUMENT} +{REQUEST} may be one of the following: + :sendline: Run the ARGUMENT followed by a line feed. + :send : send the characters in the ARGUMENT without a line feed. + :refresh : Use to catch up the screen with the shell if state gets out of sync. +Example: + :sendline ls -l +You may also leave off :command and it will be assumed. +Example: + ls -l +is equivalent to: + :sendline ls -l +""") + response.append (msg) + return '\n'.join(response) + +def parse_host_connect_string (hcs): + + """This parses a host connection string in the form + username:password@hostname:port. All fields are options expcet hostname. A + dictionary is returned with all four keys. Keys that were not included are + set to empty strings ''. Note that if your password has the '@' character + then you must backslash escape it. """ + + if '@' in hcs: + p = re.compile (r'(?P<username>[^@:]*)(:?)(?P<password>.*)(?!\\)@(?P<hostname>[^:]*):?(?P<port>[0-9]*)') + else: + p = re.compile (r'(?P<username>)(?P<password>)(?P<hostname>[^:]*):?(?P<port>[0-9]*)') + m = p.search (hcs) + d = m.groupdict() + d['password'] = d['password'].replace('\\@','@') + return d + +if __name__ == "__main__": + + try: + start_time = time.time() + print time.asctime() + main() + print time.asctime() + print "TOTAL TIME IN MINUTES:", + print (time.time() - start_time) / 60.0 + except Exception, e: + print str(e) + tb_dump = traceback.format_exc() + print str(tb_dump) + diff --git a/third_party/Python/module/pexpect-2.4/examples/cgishell.cgi b/third_party/Python/module/pexpect-2.4/examples/cgishell.cgi new file mode 100644 index 000000000000..1e3affc1cab6 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/cgishell.cgi @@ -0,0 +1,762 @@ +#!/usr/bin/python +##!/usr/bin/env python +"""CGI shell server + +This exposes a shell terminal on a web page. +It uses AJAX to send keys and receive screen updates. +The client web browser needs nothing but CSS and Javascript. + + --hostname : sets the remote host name to open an ssh connection to. + --username : sets the user name to login with + --password : (optional) sets the password to login with + --port : set the local port for the server to listen on + --watch : show the virtual screen after each client request + +This project is probably not the most security concious thing I've ever built. +This should be considered an experimental tool -- at best. +""" +import sys,os +sys.path.insert (0,os.getcwd()) # let local modules precede any installed modules +import socket, random, string, traceback, cgi, time, getopt, getpass, threading, resource, signal +import pxssh, pexpect, ANSI + +def exit_with_usage(exit_code=1): + print globals()['__doc__'] + os._exit(exit_code) + +def client (command, host='localhost', port=-1): + """This sends a request to the server and returns the response. + If port <= 0 then host is assumed to be the filename of a Unix domain socket. + If port > 0 then host is an inet hostname. + """ + if port <= 0: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.connect(host) + else: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((host, port)) + s.send(command) + data = s.recv (2500) + s.close() + return data + +def server (hostname, username, password, socket_filename='/tmp/server_sock', daemon_mode = True, verbose=False): + """This starts and services requests from a client. + If daemon_mode is True then this forks off a separate daemon process and returns the daemon's pid. + If daemon_mode is False then this does not return until the server is done. + """ + if daemon_mode: + mypid_name = '/tmp/%d.pid' % os.getpid() + daemon_pid = daemonize(daemon_pid_filename=mypid_name) + time.sleep(1) + if daemon_pid != 0: + os.unlink(mypid_name) + return daemon_pid + + virtual_screen = ANSI.ANSI (24,80) + child = pxssh.pxssh() + try: + child.login (hostname, username, password, login_naked=True) + except: + return + if verbose: print 'login OK' + virtual_screen.write (child.before) + virtual_screen.write (child.after) + + if os.path.exists(socket_filename): os.remove(socket_filename) + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.bind(socket_filename) + os.chmod(socket_filename, 0777) + if verbose: print 'Listen' + s.listen(1) + + r = roller (endless_poll, (child, child.PROMPT, virtual_screen)) + r.start() + if verbose: print "started screen-poll-updater in background thread" + sys.stdout.flush() + try: + while True: + conn, addr = s.accept() + if verbose: print 'Connected by', addr + data = conn.recv(1024) + request = data.split(' ', 1) + if len(request)>1: + cmd = request[0].strip() + arg = request[1].strip() + else: + cmd = request[0].strip() + arg = '' + + if cmd == 'exit': + r.cancel() + break + elif cmd == 'sendline': + child.sendline (arg) + time.sleep(0.1) + shell_window = str(virtual_screen) + elif cmd == 'send' or cmd=='xsend': + if cmd=='xsend': + arg = arg.decode("hex") + child.send (arg) + time.sleep(0.1) + shell_window = str(virtual_screen) + elif cmd == 'cursor': + shell_window = '%x,%x' % (virtual_screen.cur_r, virtual_screen.cur_c) + elif cmd == 'refresh': + shell_window = str(virtual_screen) + elif cmd == 'hash': + shell_window = str(hash(str(virtual_screen))) + + response = [] + response.append (shell_window) + if verbose: print '\n'.join(response) + sent = conn.send('\n'.join(response)) + if sent < len (response): + if verbose: print "Sent is too short. Some data was cut off." + conn.close() + except e: + pass + r.cancel() + if verbose: print "cleaning up socket" + s.close() + if os.path.exists(socket_filename): os.remove(socket_filename) + if verbose: print "server done!" + +class roller (threading.Thread): + """This class continuously loops a function in a thread. + This is basically a thin layer around Thread with a + while loop and a cancel. + """ + def __init__(self, function, args=[], kwargs={}): + threading.Thread.__init__(self) + self.function = function + self.args = args + self.kwargs = kwargs + self.finished = threading.Event() + def cancel(self): + """Stop the roller.""" + self.finished.set() + def run(self): + while not self.finished.isSet(): + self.function(*self.args, **self.kwargs) + +def endless_poll (child, prompt, screen, refresh_timeout=0.1): + """This keeps the screen updated with the output of the child. + This will be run in a separate thread. See roller class. + """ + #child.logfile_read = screen + try: + s = child.read_nonblocking(4000, 0.1) + screen.write(s) + except: + pass + +def daemonize (stdin=None, stdout=None, stderr=None, daemon_pid_filename=None): + """This runs the current process in the background as a daemon. + The arguments stdin, stdout, stderr allow you to set the filename that the daemon reads and writes to. + If they are set to None then all stdio for the daemon will be directed to /dev/null. + If daemon_pid_filename is set then the pid of the daemon will be written to it as plain text + and the pid will be returned. If daemon_pid_filename is None then this will return None. + """ + UMASK = 0 + WORKINGDIR = "/" + MAXFD = 1024 + + # The stdio file descriptors are redirected to /dev/null by default. + if hasattr(os, "devnull"): + DEVNULL = os.devnull + else: + DEVNULL = "/dev/null" + if stdin is None: stdin = DEVNULL + if stdout is None: stdout = DEVNULL + if stderr is None: stderr = DEVNULL + + try: + pid = os.fork() + except OSError, e: + raise Exception, "%s [%d]" % (e.strerror, e.errno) + + if pid != 0: # The first child. + os.waitpid(pid,0) + if daemon_pid_filename is not None: + daemon_pid = int(file(daemon_pid_filename,'r').read()) + return daemon_pid + else: + return None + + # first child + os.setsid() + signal.signal(signal.SIGHUP, signal.SIG_IGN) + + try: + pid = os.fork() # fork second child + except OSError, e: + raise Exception, "%s [%d]" % (e.strerror, e.errno) + + if pid != 0: + if daemon_pid_filename is not None: + file(daemon_pid_filename,'w').write(str(pid)) + os._exit(0) # exit parent (the first child) of the second child. + + # second child + os.chdir(WORKINGDIR) + os.umask(UMASK) + + maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + if maxfd == resource.RLIM_INFINITY: + maxfd = MAXFD + + # close all file descriptors + for fd in xrange(0, maxfd): + try: + os.close(fd) + except OSError: # fd wasn't open to begin with (ignored) + pass + + os.open (DEVNULL, os.O_RDWR) # standard input + + # redirect standard file descriptors + si = open(stdin, 'r') + so = open(stdout, 'a+') + se = open(stderr, 'a+', 0) + os.dup2(si.fileno(), sys.stdin.fileno()) + os.dup2(so.fileno(), sys.stdout.fileno()) + os.dup2(se.fileno(), sys.stderr.fileno()) + + return 0 + +def client_cgi (): + """This handles the request if this script was called as a cgi. + """ + sys.stderr = sys.stdout + ajax_mode = False + TITLE="Shell" + SHELL_OUTPUT="" + SID="NOT" + print "Content-type: text/html;charset=utf-8\r\n" + try: + form = cgi.FieldStorage() + if form.has_key('ajax'): + ajax_mode = True + ajax_cmd = form['ajax'].value + SID=form['sid'].value + if ajax_cmd == 'send': + command = 'xsend' + arg = form['arg'].value.encode('hex') + result = client (command + ' ' + arg, '/tmp/'+SID) + print result + elif ajax_cmd == 'refresh': + command = 'refresh' + result = client (command, '/tmp/'+SID) + print result + elif ajax_cmd == 'cursor': + command = 'cursor' + result = client (command, '/tmp/'+SID) + print result + elif ajax_cmd == 'exit': + command = 'exit' + result = client (command, '/tmp/'+SID) + print result + elif ajax_cmd == 'hash': + command = 'hash' + result = client (command, '/tmp/'+SID) + print result + elif not form.has_key('sid'): + SID=random_sid() + print LOGIN_HTML % locals(); + else: + SID=form['sid'].value + if form.has_key('start_server'): + USERNAME = form['username'].value + PASSWORD = form['password'].value + dpid = server ('127.0.0.1', USERNAME, PASSWORD, '/tmp/'+SID) + SHELL_OUTPUT="daemon pid: " + str(dpid) + else: + if form.has_key('cli'): + command = 'sendline ' + form['cli'].value + else: + command = 'sendline' + SHELL_OUTPUT = client (command, '/tmp/'+SID) + print CGISH_HTML % locals() + except: + tb_dump = traceback.format_exc() + if ajax_mode: + print str(tb_dump) + else: + SHELL_OUTPUT=str(tb_dump) + print CGISH_HTML % locals() + +def server_cli(): + """This is the command line interface to starting the server. + This handles things if the script was not called as a CGI + (if you run it from the command line). + """ + try: + optlist, args = getopt.getopt(sys.argv[1:], 'h?d', ['help','h','?', 'hostname=', 'username=', 'password=', 'port=', 'watch']) + except Exception, e: + print str(e) + exit_with_usage() + + command_line_options = dict(optlist) + options = dict(optlist) + # There are a million ways to cry for help. These are but a few of them. + if [elem for elem in command_line_options if elem in ['-h','--h','-?','--?','--help']]: + exit_with_usage(0) + + hostname = "127.0.0.1" + #port = 1664 + username = os.getenv('USER') + password = "" + daemon_mode = False + if '-d' in options: + daemon_mode = True + if '--watch' in options: + watch_mode = True + else: + watch_mode = False + if '--hostname' in options: + hostname = options['--hostname'] + if '--port' in options: + port = int(options['--port']) + if '--username' in options: + username = options['--username'] + if '--password' in options: + password = options['--password'] + else: + password = getpass.getpass('password: ') + + server (hostname, username, password, '/tmp/mysock', daemon_mode) + +def random_sid (): + a=random.randint(0,65535) + b=random.randint(0,65535) + return '%04x%04x.sid' % (a,b) + +def parse_host_connect_string (hcs): + """This parses a host connection string in the form + username:password@hostname:port. All fields are options expcet hostname. A + dictionary is returned with all four keys. Keys that were not included are + set to empty strings ''. Note that if your password has the '@' character + then you must backslash escape it. + """ + if '@' in hcs: + p = re.compile (r'(?P<username>[^@:]*)(:?)(?P<password>.*)(?!\\)@(?P<hostname>[^:]*):?(?P<port>[0-9]*)') + else: + p = re.compile (r'(?P<username>)(?P<password>)(?P<hostname>[^:]*):?(?P<port>[0-9]*)') + m = p.search (hcs) + d = m.groupdict() + d['password'] = d['password'].replace('\\@','@') + return d + +def pretty_box (s, rows=24, cols=80): + """This puts an ASCII text box around the given string. + """ + top_bot = '+' + '-'*cols + '+\n' + return top_bot + '\n'.join(['|'+line+'|' for line in s.split('\n')]) + '\n' + top_bot + +def main (): + if os.getenv('REQUEST_METHOD') is None: + server_cli() + else: + client_cgi() + +# It's mostly HTML and Javascript from here on out. +CGISH_HTML="""<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html> +<head> +<title>%(TITLE)s %(SID)s</title> +<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> +<style type=text/css> +a {color: #9f9; text-decoration: none} +a:hover {color: #0f0} +hr {color: #0f0} +html,body,textarea,input,form +{ +font-family: "Courier New", Courier, mono; +font-size: 8pt; +color: #0c0; +background-color: #020; +margin:0; +padding:0; +border:0; +} +input { background-color: #010; } +textarea { +border-width:1; +border-style:solid; +border-color:#0c0; +padding:3; +margin:3; +} +</style> + +<script language="JavaScript"> +function focus_first() +{if (document.forms.length > 0) +{var TForm = document.forms[0]; +for (i=0;i<TForm.length;i++){ +if ((TForm.elements[i].type=="text")|| +(TForm.elements[i].type=="textarea")|| +(TForm.elements[i].type.toString().charAt(0)=="s")) +{document.forms[0].elements[i].focus();break;}}}} + +// JavaScript Virtual Keyboard +// If you like this code then buy me a sandwich. +// Noah Spurrier <noah@noah.org> +var flag_shift=0; +var flag_shiftlock=0; +var flag_ctrl=0; +var ButtonOnColor="#ee0"; + +function init () +{ + // hack to set quote key to show both single quote and double quote + document.form['quote'].value = "'" + ' "'; + //refresh_screen(); + poll(); + document.form["cli"].focus(); +} +function get_password () +{ + var username = prompt("username?",""); + var password = prompt("password?",""); + start_server (username, password); +} +function multibrowser_ajax () +{ + var xmlHttp = false; +/*@cc_on @*/ +/*@if (@_jscript_version >= 5) + try + { + xmlHttp = new ActiveXObject("Msxml2.XMLHTTP"); + } + catch (e) + { + try + { + xmlHttp = new ActiveXObject("Microsoft.XMLHTTP"); + } + catch (e2) + { + xmlHttp = false; + } + } +@end @*/ + + if (!xmlHttp && typeof XMLHttpRequest != 'undefined') + { + xmlHttp = new XMLHttpRequest(); + } + return xmlHttp; +} +function load_url_to_screen(url) +{ + xmlhttp = multibrowser_ajax(); + //window.XMLHttpRequest?new XMLHttpRequest(): new ActiveXObject("Microsoft.XMLHTTP"); + xmlhttp.onreadystatechange = update_virtual_screen; + xmlhttp.open("GET", url); + xmlhttp.setRequestHeader("If-Modified-Since", "Sat, 1 Jan 2000 00:00:00 GMT"); + xmlhttp.send(null); +} +function update_virtual_screen() +{ + if ((xmlhttp.readyState == 4) && (xmlhttp.status == 200)) + { + var screen_text = xmlhttp.responseText; + document.form["screen_text"].value = screen_text; + //var json_data = json_parse(xmlhttp.responseText); + } +} +function poll() +{ + refresh_screen(); + timerID = setTimeout("poll()", 2000); + // clearTimeout(timerID); +} +//function start_server (username, password) +//{ +// load_url_to_screen('cgishell.cgi?ajax=serverstart&username=' + escape(username) + '&password=' + escape(password); +//} +function refresh_screen() +{ + load_url_to_screen('cgishell.cgi?ajax=refresh&sid=%(SID)s'); +} +function query_hash() +{ + load_url_to_screen('cgishell.cgi?ajax=hash&sid=%(SID)s'); +} +function query_cursor() +{ + load_url_to_screen('cgishell.cgi?ajax=cursor&sid=%(SID)s'); +} +function exit_server() +{ + load_url_to_screen('cgishell.cgi?ajax=exit&sid=%(SID)s'); +} +function type_key (chars) +{ + var ch = '?'; + if (flag_shiftlock || flag_shift) + { + ch = chars.substr(1,1); + } + else if (flag_ctrl) + { + ch = chars.substr(2,1); + } + else + { + ch = chars.substr(0,1); + } + load_url_to_screen('cgishell.cgi?ajax=send&sid=%(SID)s&arg=' + escape(ch)); + if (flag_shift || flag_ctrl) + { + flag_shift = 0; + flag_ctrl = 0; + } + update_button_colors(); +} + +function key_shiftlock() +{ + flag_ctrl = 0; + flag_shift = 0; + if (flag_shiftlock) + { + flag_shiftlock = 0; + } + else + { + flag_shiftlock = 1; + } + update_button_colors(); +} + +function key_shift() +{ + if (flag_shift) + { + flag_shift = 0; + } + else + { + flag_ctrl = 0; + flag_shiftlock = 0; + flag_shift = 1; + } + update_button_colors(); +} +function key_ctrl () +{ + if (flag_ctrl) + { + flag_ctrl = 0; + } + else + { + flag_ctrl = 1; + flag_shiftlock = 0; + flag_shift = 0; + } + + update_button_colors(); +} +function update_button_colors () +{ + if (flag_ctrl) + { + document.form['Ctrl'].style.backgroundColor = ButtonOnColor; + document.form['Ctrl2'].style.backgroundColor = ButtonOnColor; + } + else + { + document.form['Ctrl'].style.backgroundColor = document.form.style.backgroundColor; + document.form['Ctrl2'].style.backgroundColor = document.form.style.backgroundColor; + } + if (flag_shift) + { + document.form['Shift'].style.backgroundColor = ButtonOnColor; + document.form['Shift2'].style.backgroundColor = ButtonOnColor; + } + else + { + document.form['Shift'].style.backgroundColor = document.form.style.backgroundColor; + document.form['Shift2'].style.backgroundColor = document.form.style.backgroundColor; + } + if (flag_shiftlock) + { + document.form['ShiftLock'].style.backgroundColor = ButtonOnColor; + } + else + { + document.form['ShiftLock'].style.backgroundColor = document.form.style.backgroundColor; + } + +} +function keyHandler(e) +{ + var pressedKey; + if (document.all) { e = window.event; } + if (document.layers) { pressedKey = e.which; } + if (document.all) { pressedKey = e.keyCode; } + pressedCharacter = String.fromCharCode(pressedKey); + type_key(pressedCharacter+pressedCharacter+pressedCharacter); + alert(pressedCharacter); +// alert(' Character = ' + pressedCharacter + ' [Decimal value = ' + pressedKey + ']'); +} +//document.onkeypress = keyHandler; +//if (document.layers) +// document.captureEvents(Event.KEYPRESS); +//http://sniptools.com/jskeys +//document.onkeyup = KeyCheck; +function KeyCheck(e) +{ + var KeyID = (window.event) ? event.keyCode : e.keyCode; + type_key(String.fromCharCode(KeyID)); + e.cancelBubble = true; + window.event.cancelBubble = true; +} +</script> + +</head> + +<body onload="init()"> +<form id="form" name="form" action="/cgi-bin/cgishell.cgi" method="POST"> +<input name="sid" value="%(SID)s" type="hidden"> +<textarea name="screen_text" cols="81" rows="25">%(SHELL_OUTPUT)s</textarea> +<hr noshade="1"> + <input name="cli" id="cli" type="text" size="80"><br> +<table border="0" align="left"> +<tr> +<td width="86%%" align="center"> + <input name="submit" type="submit" value="Submit"> + <input name="refresh" type="button" value="REFRESH" onclick="refresh_screen()"> + <input name="refresh" type="button" value="CURSOR" onclick="query_cursor()"> + <input name="hash" type="button" value="HASH" onclick="query_hash()"> + <input name="exit" type="button" value="EXIT" onclick="exit_server()"> + <br> + <input type="button" value="Esc" onclick="type_key('\\x1b\\x1b')" /> + <input type="button" value="` ~" onclick="type_key('`~')" /> + <input type="button" value="1!" onclick="type_key('1!')" /> + <input type="button" value="2@" onclick="type_key('2@\\x00')" /> + <input type="button" value="3#" onclick="type_key('3#')" /> + <input type="button" value="4$" onclick="type_key('4$')" /> + <input type="button" value="5%%" onclick="type_key('5%%')" /> + <input type="button" value="6^" onclick="type_key('6^\\x1E')" /> + <input type="button" value="7&" onclick="type_key('7&')" /> + <input type="button" value="8*" onclick="type_key('8*')" /> + <input type="button" value="9(" onclick="type_key('9(')" /> + <input type="button" value="0)" onclick="type_key('0)')" /> + <input type="button" value="-_" onclick="type_key('-_\\x1F')" /> + <input type="button" value="=+" onclick="type_key('=+')" /> + <input type="button" value="BkSp" onclick="type_key('\\x08\\x08\\x08')" /> + <br> + <input type="button" value="Tab" onclick="type_key('\\t\\t')" /> + <input type="button" value="Q" onclick="type_key('qQ\\x11')" /> + <input type="button" value="W" onclick="type_key('wW\\x17')" /> + <input type="button" value="E" onclick="type_key('eE\\x05')" /> + <input type="button" value="R" onclick="type_key('rR\\x12')" /> + <input type="button" value="T" onclick="type_key('tT\\x14')" /> + <input type="button" value="Y" onclick="type_key('yY\\x19')" /> + <input type="button" value="U" onclick="type_key('uU\\x15')" /> + <input type="button" value="I" onclick="type_key('iI\\x09')" /> + <input type="button" value="O" onclick="type_key('oO\\x0F')" /> + <input type="button" value="P" onclick="type_key('pP\\x10')" /> + <input type="button" value="[ {" onclick="type_key('[{\\x1b')" /> + <input type="button" value="] }" onclick="type_key(']}\\x1d')" /> + <input type="button" value="\\ |" onclick="type_key('\\\\|\\x1c')" /> + <br> + <input type="button" id="Ctrl" value="Ctrl" onclick="key_ctrl()" /> + <input type="button" value="A" onclick="type_key('aA\\x01')" /> + <input type="button" value="S" onclick="type_key('sS\\x13')" /> + <input type="button" value="D" onclick="type_key('dD\\x04')" /> + <input type="button" value="F" onclick="type_key('fF\\x06')" /> + <input type="button" value="G" onclick="type_key('gG\\x07')" /> + <input type="button" value="H" onclick="type_key('hH\\x08')" /> + <input type="button" value="J" onclick="type_key('jJ\\x0A')" /> + <input type="button" value="K" onclick="type_key('kK\\x0B')" /> + <input type="button" value="L" onclick="type_key('lL\\x0C')" /> + <input type="button" value="; :" onclick="type_key(';:')" /> + <input type="button" id="quote" value="'" onclick="type_key('\\x27\\x22')" /> + <input type="button" value="Enter" onclick="type_key('\\n\\n')" /> + <br> + <input type="button" id="ShiftLock" value="Caps Lock" onclick="key_shiftlock()" /> + <input type="button" id="Shift" value="Shift" onclick="key_shift()" /> + <input type="button" value="Z" onclick="type_key('zZ\\x1A')" /> + <input type="button" value="X" onclick="type_key('xX\\x18')" /> + <input type="button" value="C" onclick="type_key('cC\\x03')" /> + <input type="button" value="V" onclick="type_key('vV\\x16')" /> + <input type="button" value="B" onclick="type_key('bB\\x02')" /> + <input type="button" value="N" onclick="type_key('nN\\x0E')" /> + <input type="button" value="M" onclick="type_key('mM\\x0D')" /> + <input type="button" value=", <" onclick="type_key(',<')" /> + <input type="button" value=". >" onclick="type_key('.>')" /> + <input type="button" value="/ ?" onclick="type_key('/?')" /> + <input type="button" id="Shift2" value="Shift" onclick="key_shift()" /> + <input type="button" id="Ctrl2" value="Ctrl" onclick="key_ctrl()" /> + <br> + <input type="button" value=" FINAL FRONTIER " onclick="type_key(' ')" /> +</td> +</tr> +</table> +</form> +</body> +</html> +""" + +LOGIN_HTML="""<html> +<head> +<title>Shell Login</title> +<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> +<style type=text/css> +a {color: #9f9; text-decoration: none} +a:hover {color: #0f0} +hr {color: #0f0} +html,body,textarea,input,form +{ +font-family: "Courier New", Courier, mono; +font-size: 8pt; +color: #0c0; +background-color: #020; +margin:3; +padding:0; +border:0; +} +input { background-color: #010; } +input,textarea { +border-width:1; +border-style:solid; +border-color:#0c0; +padding:3; +margin:3; +} +</style> +<script language="JavaScript"> +function init () +{ + document.login_form["username"].focus(); +} +</script> +</head> +<body onload="init()"> +<form name="login_form" method="POST"> +<input name="start_server" value="1" type="hidden"> +<input name="sid" value="%(SID)s" type="hidden"> +username: <input name="username" type="text" size="30"><br> +password: <input name="password" type="password" size="30"><br> +<input name="submit" type="submit" value="enter"> +</form> +<br> +</body> +</html> +""" + +if __name__ == "__main__": + try: + main() + except Exception, e: + print str(e) + tb_dump = traceback.format_exc() + print str(tb_dump) + diff --git a/third_party/Python/module/pexpect-2.4/examples/chess.py b/third_party/Python/module/pexpect-2.4/examples/chess.py new file mode 100644 index 000000000000..8c32cf798f2a --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/chess.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +'''This demonstrates controlling a screen oriented application (curses). +It starts two instances of gnuchess and then pits them against each other. +''' + +import pexpect +import string +import ANSI + +REGEX_MOVE = '(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)' +REGEX_MOVE_PART = '(?:[0-9]|\x1b\[C)(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)' + +class Chess: + + def __init__(self, engine = "/usr/local/bin/gnuchess -a -h 1"): + self.child = pexpect.spawn (engine) + self.term = ANSI.ANSI () + + self.child.expect ('Chess') + if self.child.after != 'Chess': + raise IOError, 'incompatible chess program' + self.term.process_list (self.before) + self.term.process_list (self.after) + self.last_computer_move = '' + def read_until_cursor (self, r,c) + while 1: + self.child.read(1, 60) + self.term.process (c) + if self.term.cur_r == r and self.term.cur_c == c: + return 1 + + def do_first_move (self, move): + self.child.expect ('Your move is') + self.child.sendline (move) + self.term.process_list (self.before) + self.term.process_list (self.after) + return move + + def do_move (self, move): + read_until_cursor (19,60) + #self.child.expect ('\[19;60H') + self.child.sendline (move) + print 'do_move' move + return move + + def get_first_computer_move (self): + self.child.expect ('My move is') + self.child.expect (REGEX_MOVE) +# print '', self.child.after + return self.child.after + + def get_computer_move (self): + print 'Here' + i = self.child.expect (['\[17;59H', '\[17;58H']) + print i + if i == 0: + self.child.expect (REGEX_MOVE) + if len(self.child.after) < 4: + self.child.after = self.child.after + self.last_computer_move[3] + if i == 1: + self.child.expect (REGEX_MOVE_PART) + self.child.after = self.last_computer_move[0] + self.child.after + print '', self.child.after + self.last_computer_move = self.child.after + return self.child.after + + def switch (self): + self.child.sendline ('switch') + + def set_depth (self, depth): + self.child.sendline ('depth') + self.child.expect ('depth=') + self.child.sendline ('%d' % depth) + + def quit(self): + self.child.sendline ('quit') +import sys, os +print 'Starting...' +white = Chess() +white.child.echo = 1 +white.child.expect ('Your move is') +white.set_depth(2) +white.switch() + +move_white = white.get_first_computer_move() +print 'first move white:', move_white + +white.do_move ('e7e5') +move_white = white.get_computer_move() +print 'move white:', move_white +white.do_move ('f8c5') +move_white = white.get_computer_move() +print 'move white:', move_white +white.do_move ('b8a6') +move_white = white.get_computer_move() +print 'move white:', move_white + +sys.exit(1) + + + +black = Chess() +white = Chess() +white.child.expect ('Your move is') +white.switch() + +move_white = white.get_first_computer_move() +print 'first move white:', move_white + +black.do_first_move (move_white) +move_black = black.get_first_computer_move() +print 'first move black:', move_black + +white.do_move (move_black) + +done = 0 +while not done: + move_white = white.get_computer_move() + print 'move white:', move_white + + black.do_move (move_white) + move_black = black.get_computer_move() + print 'move black:', move_black + + white.do_move (move_black) + print 'tail of loop' + +g.quit() + + diff --git a/third_party/Python/module/pexpect-2.4/examples/chess2.py b/third_party/Python/module/pexpect-2.4/examples/chess2.py new file mode 100644 index 000000000000..c62d5ce37a06 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/chess2.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +'''This demonstrates controlling a screen oriented application (curses). +It starts two instances of gnuchess and then pits them against each other. +''' + +import pexpect +import string +import ANSI +import sys, os, time + +class Chess: + + def __init__(self, engine = "/usr/local/bin/gnuchess -a -h 1"): + self.child = pexpect.spawn (engine) + self.term = ANSI.ANSI () + + #self.child.expect ('Chess') + #if self.child.after != 'Chess': + # raise IOError, 'incompatible chess program' + #self.term.process_list (self.child.before) + #self.term.process_list (self.child.after) + + self.last_computer_move = '' + + def read_until_cursor (self, r,c, e=0): + '''Eventually something like this should move into the screen class or + a subclass. Maybe a combination of pexpect and screen... + ''' + fout = open ('log','a') + while self.term.cur_r != r or self.term.cur_c != c: + try: + k = self.child.read(1, 10) + except Exception, e: + print 'EXCEPTION, (r,c):(%d,%d)\n' %(self.term.cur_r, self.term.cur_c) + sys.stdout.flush() + self.term.process (k) + fout.write ('(r,c):(%d,%d)\n' %(self.term.cur_r, self.term.cur_c)) + fout.flush() + if e: + sys.stdout.write (k) + sys.stdout.flush() + if self.term.cur_r == r and self.term.cur_c == c: + fout.close() + return 1 + print 'DIDNT EVEN HIT.' + fout.close() + return 1 + + def expect_region (self): + '''This is another method that would be moved into the + screen class. + ''' + pass + def do_scan (self): + fout = open ('log','a') + while 1: + c = self.child.read(1,10) + self.term.process (c) + fout.write ('(r,c):(%d,%d)\n' %(self.term.cur_r, self.term.cur_c)) + fout.flush() + sys.stdout.write (c) + sys.stdout.flush() + + def do_move (self, move, e = 0): + time.sleep(1) + self.read_until_cursor (19,60, e) + self.child.sendline (move) + + def wait (self, color): + while 1: + r = self.term.get_region (14,50,14,60)[0] + r = r.strip() + if r == color: + return + time.sleep (1) + + def parse_computer_move (self, s): + i = s.find ('is: ') + cm = s[i+3:i+9] + return cm + def get_computer_move (self, e = 0): + time.sleep(1) + self.read_until_cursor (19,60, e) + time.sleep(1) + r = self.term.get_region (17,50,17,62)[0] + cm = self.parse_computer_move (r) + return cm + + def switch (self): + print 'switching' + self.child.sendline ('switch') + + def set_depth (self, depth): + self.child.sendline ('depth') + self.child.expect ('depth=') + self.child.sendline ('%d' % depth) + + def quit(self): + self.child.sendline ('quit') + +def LOG (s): + print s + sys.stdout.flush () + fout = open ('moves.log', 'a') + fout.write (s + '\n') + fout.close() + +print 'Starting...' + +black = Chess() +white = Chess() +white.read_until_cursor (19,60,1) +white.switch() + +done = 0 +while not done: + white.wait ('Black') + move_white = white.get_computer_move(1) + LOG ( 'move white:'+ move_white ) + + black.do_move (move_white) + black.wait ('White') + move_black = black.get_computer_move() + LOG ( 'move black:'+ move_black ) + + white.do_move (move_black, 1) + +g.quit() + + diff --git a/third_party/Python/module/pexpect-2.4/examples/chess3.py b/third_party/Python/module/pexpect-2.4/examples/chess3.py new file mode 100644 index 000000000000..44044420a1c9 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/chess3.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python + +'''This demonstrates controlling a screen oriented application (curses). +It starts two instances of gnuchess and then pits them against each other. +''' + +import pexpect +import string +import ANSI + +REGEX_MOVE = '(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)' +REGEX_MOVE_PART = '(?:[0-9]|\x1b\[C)(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)' + +class Chess: + + def __init__(self, engine = "/usr/local/bin/gnuchess -a -h 1"): + self.child = pexpect.spawn (engine) + self.term = ANSI.ANSI () + +# self.child.expect ('Chess') + # if self.child.after != 'Chess': + # raise IOError, 'incompatible chess program' + # self.term.process_list (self.before) + # self.term.process_list (self.after) + self.last_computer_move = '' + def read_until_cursor (self, r,c): + fout = open ('log','a') + while 1: + k = self.child.read(1, 10) + self.term.process (k) + fout.write ('(r,c):(%d,%d)\n' %(self.term.cur_r, self.term.cur_c)) + fout.flush() + if self.term.cur_r == r and self.term.cur_c == c: + fout.close() + return 1 + sys.stdout.write (k) + sys.stdout.flush() + + def do_scan (self): + fout = open ('log','a') + while 1: + c = self.child.read(1,10) + self.term.process (c) + fout.write ('(r,c):(%d,%d)\n' %(self.term.cur_r, self.term.cur_c)) + fout.flush() + sys.stdout.write (c) + sys.stdout.flush() + + def do_move (self, move): + self.read_until_cursor (19,60) + self.child.sendline (move) + return move + + def get_computer_move (self): + print 'Here' + i = self.child.expect (['\[17;59H', '\[17;58H']) + print i + if i == 0: + self.child.expect (REGEX_MOVE) + if len(self.child.after) < 4: + self.child.after = self.child.after + self.last_computer_move[3] + if i == 1: + self.child.expect (REGEX_MOVE_PART) + self.child.after = self.last_computer_move[0] + self.child.after + print '', self.child.after + self.last_computer_move = self.child.after + return self.child.after + + def switch (self): + self.child.sendline ('switch') + + def set_depth (self, depth): + self.child.sendline ('depth') + self.child.expect ('depth=') + self.child.sendline ('%d' % depth) + + def quit(self): + self.child.sendline ('quit') +import sys, os +print 'Starting...' +white = Chess() +white.do_move('b2b4') +white.read_until_cursor (19,60) +c1 = white.term.get_abs(17,58) +c2 = white.term.get_abs(17,59) +c3 = white.term.get_abs(17,60) +c4 = white.term.get_abs(17,61) +fout = open ('log','a') +fout.write ('Computer:%s%s%s%s\n' %(c1,c2,c3,c4)) +fout.close() +white.do_move('c2c4') +white.read_until_cursor (19,60) +c1 = white.term.get_abs(17,58) +c2 = white.term.get_abs(17,59) +c3 = white.term.get_abs(17,60) +c4 = white.term.get_abs(17,61) +fout = open ('log','a') +fout.write ('Computer:%s%s%s%s\n' %(c1,c2,c3,c4)) +fout.close() +white.do_scan () + +#white.do_move ('b8a6') +#move_white = white.get_computer_move() +#print 'move white:', move_white + +sys.exit(1) + + + +black = Chess() +white = Chess() +white.child.expect ('Your move is') +white.switch() + +move_white = white.get_first_computer_move() +print 'first move white:', move_white + +black.do_first_move (move_white) +move_black = black.get_first_computer_move() +print 'first move black:', move_black + +white.do_move (move_black) + +done = 0 +while not done: + move_white = white.get_computer_move() + print 'move white:', move_white + + black.do_move (move_white) + move_black = black.get_computer_move() + print 'move black:', move_black + + white.do_move (move_black) + print 'tail of loop' + +g.quit() + + diff --git a/third_party/Python/module/pexpect-2.4/examples/df.py b/third_party/Python/module/pexpect-2.4/examples/df.py new file mode 100644 index 000000000000..64bbf93a1b29 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/df.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python + +"""This collects filesystem capacity info using the 'df' command. Tuples of +filesystem name and percentage are stored in a list. A simple report is +printed. Filesystems over 95% capacity are highlighted. Note that this does not +parse filesystem names after the first space, so names with spaces in them will +be truncated. This will produce ambiguous results for automount filesystems on +Apple OSX. """ + +import pexpect + +child = pexpect.spawn ('df') + +# parse 'df' output into a list. +pattern = "\n(\S+).*?([0-9]+)%" +filesystem_list = [] +for dummy in range (0, 1000): + i = child.expect ([pattern, pexpect.EOF]) + if i == 0: + filesystem_list.append (child.match.groups()) + else: + break + +# Print report +print +for m in filesystem_list: + s = "Filesystem %s is at %s%%" % (m[0], m[1]) + # highlight filesystems over 95% capacity + if int(m[1]) > 95: + s = '! ' + s + else: + s = ' ' + s + print s + diff --git a/third_party/Python/module/pexpect-2.4/examples/fix_cvs_files.py b/third_party/Python/module/pexpect-2.4/examples/fix_cvs_files.py new file mode 100644 index 000000000000..e75a1496abdd --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/fix_cvs_files.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +"""This is for cleaning up binary files improperly added to CVS. This script +scans the given path to find binary files; checks with CVS to see if the sticky +options are set to -kb; finally if sticky options are not -kb then uses 'cvs +admin' to set the -kb option. + +This script ignores CVS directories, symbolic links, and files not known under +CVS control (cvs status is 'Unknown'). + +Run this on a CHECKED OUT module sandbox, not on the repository itself. After +if fixes the sticky options on any files you should manually do a 'cvs commit' +to accept the changes. Then be sure to have all users do a 'cvs up -A' to +update the Sticky Option status. + +Noah Spurrier +20030426 +""" + +import os, sys, time +import pexpect + +VERBOSE = 1 + +def is_binary (filename): + + """Assume that any file with a character where the 8th bit is set is + binary. """ + + fin = open(filename, 'rb') + wholething = fin.read() + fin.close() + for c in wholething: + if ord(c) & 0x80: + return 1 + return 0 + +def is_kb_sticky (filename): + + """This checks if 'cvs status' reports '-kb' for Sticky options. If the + Sticky Option status is '-ks' then this returns 1. If the status is + 'Unknown' then it returns 1. Otherwise 0 is returned. """ + + try: + s = pexpect.spawn ('cvs status %s' % filename) + i = s.expect (['Sticky Options:\s*(.*)\r\n', 'Status: Unknown']) + if i==1 and VERBOSE: + print 'File not part of CVS repository:', filename + return 1 # Pretend it's OK. + if s.match.group(1) == '-kb': + return 1 + s = None + except: + print 'Something went wrong trying to run external cvs command.' + print ' cvs status %s' % filename + print 'The cvs command returned:' + print s.before + return 0 + +def cvs_admin_kb (filename): + + """This uses 'cvs admin' to set the '-kb' sticky option. """ + + s = pexpect.run ('cvs admin -kb %s' % filename) + # There is a timing issue. If I run 'cvs admin' too quickly + # cvs sometimes has trouble obtaining the directory lock. + time.sleep(1) + +def walk_and_clean_cvs_binaries (arg, dirname, names): + + """This contains the logic for processing files. This is the os.path.walk + callback. This skips dirnames that end in CVS. """ + + if len(dirname)>3 and dirname[-3:]=='CVS': + return + for n in names: + fullpath = os.path.join (dirname, n) + if os.path.isdir(fullpath) or os.path.islink(fullpath): + continue + if is_binary(fullpath): + if not is_kb_sticky (fullpath): + if VERBOSE: print fullpath + cvs_admin_kb (fullpath) + +def main (): + + if len(sys.argv) == 1: + root = '.' + else: + root = sys.argv[1] + os.path.walk (root, walk_and_clean_cvs_binaries, None) + +if __name__ == '__main__': + main () + diff --git a/third_party/Python/module/pexpect-2.4/examples/ftp.py b/third_party/Python/module/pexpect-2.4/examples/ftp.py new file mode 100644 index 000000000000..89a502e1b8f4 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/ftp.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +"""This demonstrates an FTP "bookmark". This connects to an ftp site; does a +few ftp stuff; and then gives the user interactive control over the session. In +this case the "bookmark" is to a directory on the OpenBSD ftp server. It puts +you in the i386 packages directory. You can easily modify this for other sites. +""" + +import pexpect +import sys + +child = pexpect.spawn('ftp ftp.openbsd.org') +child.expect('(?i)name .*: ') +child.sendline('anonymous') +child.expect('(?i)password') +child.sendline('pexpect@sourceforge.net') +child.expect('ftp> ') +child.sendline('cd /pub/OpenBSD/3.7/packages/i386') +child.expect('ftp> ') +child.sendline('bin') +child.expect('ftp> ') +child.sendline('prompt') +child.expect('ftp> ') +child.sendline('pwd') +child.expect('ftp> ') +print("Escape character is '^]'.\n") +sys.stdout.write (child.after) +sys.stdout.flush() +child.interact() # Escape character defaults to ^] +# At this point this script blocks until the user presses the escape character +# or until the child exits. The human user and the child should be talking +# to each other now. + +# At this point the script is running again. +print 'Left interactve mode.' + +# The rest is not strictly necessary. This just demonstrates a few functions. +# This makes sure the child is dead; although it would be killed when Python exits. +if child.isalive(): + child.sendline('bye') # Try to ask ftp child to exit. + child.close() +# Print the final state of the child. Normally isalive() should be FALSE. +if child.isalive(): + print 'Child did not exit gracefully.' +else: + print 'Child exited gracefully.' + diff --git a/third_party/Python/module/pexpect-2.4/examples/hive.py b/third_party/Python/module/pexpect-2.4/examples/hive.py new file mode 100644 index 000000000000..fcb75bcac67f --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/hive.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python + +"""hive -- Hive Shell + +This lets you ssh to a group of servers and control them as if they were one. +Each command you enter is sent to each host in parallel. The response of each +host is collected and printed. In normal synchronous mode Hive will wait for +each host to return the shell command line prompt. The shell prompt is used to +sync output. + +Example: + + $ hive.py --sameuser --samepass host1.example.com host2.example.net + username: myusername + password: + connecting to host1.example.com - OK + connecting to host2.example.net - OK + targetting hosts: 192.168.1.104 192.168.1.107 + CMD (? for help) > uptime + ======================================================================= + host1.example.com + ----------------------------------------------------------------------- + uptime + 23:49:55 up 74 days, 5:14, 2 users, load average: 0.15, 0.05, 0.01 + ======================================================================= + host2.example.net + ----------------------------------------------------------------------- + uptime + 23:53:02 up 1 day, 13:36, 2 users, load average: 0.50, 0.40, 0.46 + ======================================================================= + +Other Usage Examples: + +1. You will be asked for your username and password for each host. + + hive.py host1 host2 host3 ... hostN + +2. You will be asked once for your username and password. + This will be used for each host. + + hive.py --sameuser --samepass host1 host2 host3 ... hostN + +3. Give a username and password on the command-line: + + hive.py user1:pass2@host1 user2:pass2@host2 ... userN:passN@hostN + +You can use an extended host notation to specify username, password, and host +instead of entering auth information interactively. Where you would enter a +host name use this format: + + username:password@host + +This assumes that ':' is not part of the password. If your password contains a +':' then you can use '\\:' to indicate a ':' and '\\\\' to indicate a single +'\\'. Remember that this information will appear in the process listing. Anyone +on your machine can see this auth information. This is not secure. + +This is a crude script that begs to be multithreaded. But it serves its +purpose. + +Noah Spurrier + +$Id: hive.py 509 2008-01-05 21:27:47Z noah $ +""" + +# TODO add feature to support username:password@host combination +# TODO add feature to log each host output in separate file + +import sys, os, re, optparse, traceback, types, time, getpass +import pexpect, pxssh +import readline, atexit + +#histfile = os.path.join(os.environ["HOME"], ".hive_history") +#try: +# readline.read_history_file(histfile) +#except IOError: +# pass +#atexit.register(readline.write_history_file, histfile) + +CMD_HELP="""Hive commands are preceded by a colon : (just think of vi). + +:target name1 name2 name3 ... + + set list of hosts to target commands + +:target all + + reset list of hosts to target all hosts in the hive. + +:to name command + + send a command line to the named host. This is similar to :target, but + sends only one command and does not change the list of targets for future + commands. + +:sync + + set mode to wait for shell prompts after commands are run. This is the + default. When Hive first logs into a host it sets a special shell prompt + pattern that it can later look for to synchronize output of the hosts. If + you 'su' to another user then it can upset the synchronization. If you need + to run something like 'su' then use the following pattern: + + CMD (? for help) > :async + CMD (? for help) > sudo su - root + CMD (? for help) > :prompt + CMD (? for help) > :sync + +:async + + set mode to not expect command line prompts (see :sync). Afterwards + commands are send to target hosts, but their responses are not read back + until :sync is run. This is useful to run before commands that will not + return with the special shell prompt pattern that Hive uses to synchronize. + +:refresh + + refresh the display. This shows the last few lines of output from all hosts. + This is similar to resync, but does not expect the promt. This is useful + for seeing what hosts are doing during long running commands. + +:resync + + This is similar to :sync, but it does not change the mode. It looks for the + prompt and thus consumes all input from all targetted hosts. + +:prompt + + force each host to reset command line prompt to the special pattern used to + synchronize all the hosts. This is useful if you 'su' to a different user + where Hive would not know the prompt to match. + +:send my text + + This will send the 'my text' wihtout a line feed to the targetted hosts. + This output of the hosts is not automatically synchronized. + +:control X + + This will send the given control character to the targetted hosts. + For example, ":control c" will send ASCII 3. + +:exit + + This will exit the hive shell. + +""" + +def login (args, cli_username=None, cli_password=None): + + # I have to keep a separate list of host names because Python dicts are not ordered. + # I want to keep the same order as in the args list. + host_names = [] + hive_connect_info = {} + hive = {} + # build up the list of connection information (hostname, username, password, port) + for host_connect_string in args: + hcd = parse_host_connect_string (host_connect_string) + hostname = hcd['hostname'] + port = hcd['port'] + if port == '': + port = None + if len(hcd['username']) > 0: + username = hcd['username'] + elif cli_username is not None: + username = cli_username + else: + username = raw_input('%s username: ' % hostname) + if len(hcd['password']) > 0: + password = hcd['password'] + elif cli_password is not None: + password = cli_password + else: + password = getpass.getpass('%s password: ' % hostname) + host_names.append(hostname) + hive_connect_info[hostname] = (hostname, username, password, port) + # build up the list of hive connections using the connection information. + for hostname in host_names: + print 'connecting to', hostname + try: + fout = file("log_"+hostname, "w") + hive[hostname] = pxssh.pxssh() + hive[hostname].login(*hive_connect_info[hostname]) + print hive[hostname].before + hive[hostname].logfile = fout + print '- OK' + except Exception, e: + print '- ERROR', + print str(e) + print 'Skipping', hostname + hive[hostname] = None + return host_names, hive + +def main (): + + global options, args, CMD_HELP + + if options.sameuser: + cli_username = raw_input('username: ') + else: + cli_username = None + + if options.samepass: + cli_password = getpass.getpass('password: ') + else: + cli_password = None + + host_names, hive = login(args, cli_username, cli_password) + + synchronous_mode = True + target_hostnames = host_names[:] + print 'targetting hosts:', ' '.join(target_hostnames) + while True: + cmd = raw_input('CMD (? for help) > ') + cmd = cmd.strip() + if cmd=='?' or cmd==':help' or cmd==':h': + print CMD_HELP + continue + elif cmd==':refresh': + refresh (hive, target_hostnames, timeout=0.5) + for hostname in target_hostnames: + if hive[hostname] is None: + print '/=============================================================================' + print '| ' + hostname + ' is DEAD' + print '\\-----------------------------------------------------------------------------' + else: + print '/=============================================================================' + print '| ' + hostname + print '\\-----------------------------------------------------------------------------' + print hive[hostname].before + print '==============================================================================' + continue + elif cmd==':resync': + resync (hive, target_hostnames, timeout=0.5) + for hostname in target_hostnames: + if hive[hostname] is None: + print '/=============================================================================' + print '| ' + hostname + ' is DEAD' + print '\\-----------------------------------------------------------------------------' + else: + print '/=============================================================================' + print '| ' + hostname + print '\\-----------------------------------------------------------------------------' + print hive[hostname].before + print '==============================================================================' + continue + elif cmd==':sync': + synchronous_mode = True + resync (hive, target_hostnames, timeout=0.5) + continue + elif cmd==':async': + synchronous_mode = False + continue + elif cmd==':prompt': + for hostname in target_hostnames: + try: + if hive[hostname] is not None: + hive[hostname].set_unique_prompt() + except Exception, e: + print "Had trouble communicating with %s, so removing it from the target list." % hostname + print str(e) + hive[hostname] = None + continue + elif cmd[:5] == ':send': + cmd, txt = cmd.split(None,1) + for hostname in target_hostnames: + try: + if hive[hostname] is not None: + hive[hostname].send(txt) + except Exception, e: + print "Had trouble communicating with %s, so removing it from the target list." % hostname + print str(e) + hive[hostname] = None + continue + elif cmd[:3] == ':to': + cmd, hostname, txt = cmd.split(None,2) + if hive[hostname] is None: + print '/=============================================================================' + print '| ' + hostname + ' is DEAD' + print '\\-----------------------------------------------------------------------------' + continue + try: + hive[hostname].sendline (txt) + hive[hostname].prompt(timeout=2) + print '/=============================================================================' + print '| ' + hostname + print '\\-----------------------------------------------------------------------------' + print hive[hostname].before + except Exception, e: + print "Had trouble communicating with %s, so removing it from the target list." % hostname + print str(e) + hive[hostname] = None + continue + elif cmd[:7] == ':expect': + cmd, pattern = cmd.split(None,1) + print 'looking for', pattern + try: + for hostname in target_hostnames: + if hive[hostname] is not None: + hive[hostname].expect(pattern) + print hive[hostname].before + except Exception, e: + print "Had trouble communicating with %s, so removing it from the target list." % hostname + print str(e) + hive[hostname] = None + continue + elif cmd[:7] == ':target': + target_hostnames = cmd.split()[1:] + if len(target_hostnames) == 0 or target_hostnames[0] == all: + target_hostnames = host_names[:] + print 'targetting hosts:', ' '.join(target_hostnames) + continue + elif cmd == ':exit' or cmd == ':q' or cmd == ':quit': + break + elif cmd[:8] == ':control' or cmd[:5] == ':ctrl' : + cmd, c = cmd.split(None,1) + if ord(c)-96 < 0 or ord(c)-96 > 255: + print '/=============================================================================' + print '| Invalid character. Must be [a-zA-Z], @, [, ], \\, ^, _, or ?' + print '\\-----------------------------------------------------------------------------' + continue + for hostname in target_hostnames: + try: + if hive[hostname] is not None: + hive[hostname].sendcontrol(c) + except Exception, e: + print "Had trouble communicating with %s, so removing it from the target list." % hostname + print str(e) + hive[hostname] = None + continue + elif cmd == ':esc': + for hostname in target_hostnames: + if hive[hostname] is not None: + hive[hostname].send(chr(27)) + continue + # + # Run the command on all targets in parallel + # + for hostname in target_hostnames: + try: + if hive[hostname] is not None: + hive[hostname].sendline (cmd) + except Exception, e: + print "Had trouble communicating with %s, so removing it from the target list." % hostname + print str(e) + hive[hostname] = None + + # + # print the response for each targeted host. + # + if synchronous_mode: + for hostname in target_hostnames: + try: + if hive[hostname] is None: + print '/=============================================================================' + print '| ' + hostname + ' is DEAD' + print '\\-----------------------------------------------------------------------------' + else: + hive[hostname].prompt(timeout=2) + print '/=============================================================================' + print '| ' + hostname + print '\\-----------------------------------------------------------------------------' + print hive[hostname].before + except Exception, e: + print "Had trouble communicating with %s, so removing it from the target list." % hostname + print str(e) + hive[hostname] = None + print '==============================================================================' + +def refresh (hive, hive_names, timeout=0.5): + + """This waits for the TIMEOUT on each host. + """ + + # TODO This is ideal for threading. + for hostname in hive_names: + hive[hostname].expect([pexpect.TIMEOUT,pexpect.EOF],timeout=timeout) + +def resync (hive, hive_names, timeout=2, max_attempts=5): + + """This waits for the shell prompt for each host in an effort to try to get + them all to the same state. The timeout is set low so that hosts that are + already at the prompt will not slow things down too much. If a prompt match + is made for a hosts then keep asking until it stops matching. This is a + best effort to consume all input if it printed more than one prompt. It's + kind of kludgy. Note that this will always introduce a delay equal to the + timeout for each machine. So for 10 machines with a 2 second delay you will + get AT LEAST a 20 second delay if not more. """ + + # TODO This is ideal for threading. + for hostname in hive_names: + for attempts in xrange(0, max_attempts): + if not hive[hostname].prompt(timeout=timeout): + break + +def parse_host_connect_string (hcs): + + """This parses a host connection string in the form + username:password@hostname:port. All fields are options expcet hostname. A + dictionary is returned with all four keys. Keys that were not included are + set to empty strings ''. Note that if your password has the '@' character + then you must backslash escape it. """ + + if '@' in hcs: + p = re.compile (r'(?P<username>[^@:]*)(:?)(?P<password>.*)(?!\\)@(?P<hostname>[^:]*):?(?P<port>[0-9]*)') + else: + p = re.compile (r'(?P<username>)(?P<password>)(?P<hostname>[^:]*):?(?P<port>[0-9]*)') + m = p.search (hcs) + d = m.groupdict() + d['password'] = d['password'].replace('\\@','@') + return d + +if __name__ == '__main__': + try: + start_time = time.time() + parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), usage=globals()['__doc__'], version='$Id: hive.py 509 2008-01-05 21:27:47Z noah $',conflict_handler="resolve") + parser.add_option ('-v', '--verbose', action='store_true', default=False, help='verbose output') + parser.add_option ('--samepass', action='store_true', default=False, help='Use same password for each login.') + parser.add_option ('--sameuser', action='store_true', default=False, help='Use same username for each login.') + (options, args) = parser.parse_args() + if len(args) < 1: + parser.error ('missing argument') + if options.verbose: print time.asctime() + main() + if options.verbose: print time.asctime() + if options.verbose: print 'TOTAL TIME IN MINUTES:', + if options.verbose: print (time.time() - start_time) / 60.0 + sys.exit(0) + except KeyboardInterrupt, e: # Ctrl-C + raise e + except SystemExit, e: # sys.exit() + raise e + except Exception, e: + print 'ERROR, UNEXPECTED EXCEPTION' + print str(e) + traceback.print_exc() + os._exit(1) diff --git a/third_party/Python/module/pexpect-2.4/examples/monitor.py b/third_party/Python/module/pexpect-2.4/examples/monitor.py new file mode 100644 index 000000000000..e31b51b6d210 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/monitor.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python + +""" This runs a sequence of commands on a remote host using SSH. It runs a +simple system checks such as uptime and free to monitor the state of the remote +host. + +./monitor.py [-s server_hostname] [-u username] [-p password] + -s : hostname of the remote server to login to. + -u : username to user for login. + -p : Password to user for login. + +Example: + This will print information about the given host: + ./monitor.py -s www.example.com -u mylogin -p mypassword + +It works like this: + Login via SSH (This is the hardest part). + Run and parse 'uptime'. + Run 'iostat'. + Run 'vmstat'. + Run 'netstat' + Run 'free'. + Exit the remote host. +""" + +import os, sys, time, re, getopt, getpass +import traceback +import pexpect + +# +# Some constants. +# +COMMAND_PROMPT = '[#$] ' ### This is way too simple for industrial use -- we will change is ASAP. +TERMINAL_PROMPT = '(?i)terminal type\?' +TERMINAL_TYPE = 'vt100' +# This is the prompt we get if SSH does not have the remote host's public key stored in the cache. +SSH_NEWKEY = '(?i)are you sure you want to continue connecting' + +def exit_with_usage(): + + print globals()['__doc__'] + os._exit(1) + +def main(): + + global COMMAND_PROMPT, TERMINAL_PROMPT, TERMINAL_TYPE, SSH_NEWKEY + ###################################################################### + ## Parse the options, arguments, get ready, etc. + ###################################################################### + try: + optlist, args = getopt.getopt(sys.argv[1:], 'h?s:u:p:', ['help','h','?']) + except Exception, e: + print str(e) + exit_with_usage() + options = dict(optlist) + if len(args) > 1: + exit_with_usage() + + if [elem for elem in options if elem in ['-h','--h','-?','--?','--help']]: + print "Help:" + exit_with_usage() + + if '-s' in options: + host = options['-s'] + else: + host = raw_input('hostname: ') + if '-u' in options: + user = options['-u'] + else: + user = raw_input('username: ') + if '-p' in options: + password = options['-p'] + else: + password = getpass.getpass('password: ') + + # + # Login via SSH + # + child = pexpect.spawn('ssh -l %s %s'%(user, host)) + i = child.expect([pexpect.TIMEOUT, SSH_NEWKEY, COMMAND_PROMPT, '(?i)password']) + if i == 0: # Timeout + print 'ERROR! could not login with SSH. Here is what SSH said:' + print child.before, child.after + print str(child) + sys.exit (1) + if i == 1: # In this case SSH does not have the public key cached. + child.sendline ('yes') + child.expect ('(?i)password') + if i == 2: + # This may happen if a public key was setup to automatically login. + # But beware, the COMMAND_PROMPT at this point is very trivial and + # could be fooled by some output in the MOTD or login message. + pass + if i == 3: + child.sendline(password) + # Now we are either at the command prompt or + # the login process is asking for our terminal type. + i = child.expect ([COMMAND_PROMPT, TERMINAL_PROMPT]) + if i == 1: + child.sendline (TERMINAL_TYPE) + child.expect (COMMAND_PROMPT) + # + # Set command prompt to something more unique. + # + COMMAND_PROMPT = "\[PEXPECT\]\$ " + child.sendline ("PS1='[PEXPECT]\$ '") # In case of sh-style + i = child.expect ([pexpect.TIMEOUT, COMMAND_PROMPT], timeout=10) + if i == 0: + print "# Couldn't set sh-style prompt -- trying csh-style." + child.sendline ("set prompt='[PEXPECT]\$ '") + i = child.expect ([pexpect.TIMEOUT, COMMAND_PROMPT], timeout=10) + if i == 0: + print "Failed to set command prompt using sh or csh style." + print "Response was:" + print child.before + sys.exit (1) + + # Now we should be at the command prompt and ready to run some commands. + print '---------------------------------------' + print 'Report of commands run on remote host.' + print '---------------------------------------' + + # Run uname. + child.sendline ('uname -a') + child.expect (COMMAND_PROMPT) + print child.before + if 'linux' in child.before.lower(): + LINUX_MODE = 1 + else: + LINUX_MODE = 0 + + # Run and parse 'uptime'. + child.sendline ('uptime') + child.expect('up\s+(.*?),\s+([0-9]+) users?,\s+load averages?: ([0-9]+\.[0-9][0-9]),?\s+([0-9]+\.[0-9][0-9]),?\s+([0-9]+\.[0-9][0-9])') + duration, users, av1, av5, av15 = child.match.groups() + days = '0' + hours = '0' + mins = '0' + if 'day' in duration: + child.match = re.search('([0-9]+)\s+day',duration) + days = str(int(child.match.group(1))) + if ':' in duration: + child.match = re.search('([0-9]+):([0-9]+)',duration) + hours = str(int(child.match.group(1))) + mins = str(int(child.match.group(2))) + if 'min' in duration: + child.match = re.search('([0-9]+)\s+min',duration) + mins = str(int(child.match.group(1))) + print + print 'Uptime: %s days, %s users, %s (1 min), %s (5 min), %s (15 min)' % ( + duration, users, av1, av5, av15) + child.expect (COMMAND_PROMPT) + + # Run iostat. + child.sendline ('iostat') + child.expect (COMMAND_PROMPT) + print child.before + + # Run vmstat. + child.sendline ('vmstat') + child.expect (COMMAND_PROMPT) + print child.before + + # Run free. + if LINUX_MODE: + child.sendline ('free') # Linux systems only. + child.expect (COMMAND_PROMPT) + print child.before + + # Run df. + child.sendline ('df') + child.expect (COMMAND_PROMPT) + print child.before + + # Run lsof. + child.sendline ('lsof') + child.expect (COMMAND_PROMPT) + print child.before + +# # Run netstat +# child.sendline ('netstat') +# child.expect (COMMAND_PROMPT) +# print child.before + +# # Run MySQL show status. +# child.sendline ('mysql -p -e "SHOW STATUS;"') +# child.expect (PASSWORD_PROMPT_MYSQL) +# child.sendline (password_mysql) +# child.expect (COMMAND_PROMPT) +# print +# print child.before + + # Now exit the remote host. + child.sendline ('exit') + index = child.expect([pexpect.EOF, "(?i)there are stopped jobs"]) + if index==1: + child.sendline("exit") + child.expect(EOF) + +if __name__ == "__main__": + + try: + main() + except Exception, e: + print str(e) + traceback.print_exc() + os._exit(1) + diff --git a/third_party/Python/module/pexpect-2.4/examples/passmass.py b/third_party/Python/module/pexpect-2.4/examples/passmass.py new file mode 100644 index 000000000000..b1e17b9cb036 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/passmass.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +"""Change passwords on the named machines. passmass host1 host2 host3 . . . +Note that login shell prompt on remote machine must end in # or $. """ + +import pexpect +import sys, getpass + +USAGE = '''passmass host1 host2 host3 . . .''' +COMMAND_PROMPT = '[$#] ' +TERMINAL_PROMPT = r'Terminal type\?' +TERMINAL_TYPE = 'vt100' +SSH_NEWKEY = r'Are you sure you want to continue connecting \(yes/no\)\?' + +def login(host, user, password): + + child = pexpect.spawn('ssh -l %s %s'%(user, host)) + fout = file ("LOG.TXT","wb") + child.setlog (fout) + + i = child.expect([pexpect.TIMEOUT, SSH_NEWKEY, '[Pp]assword: ']) + if i == 0: # Timeout + print 'ERROR!' + print 'SSH could not login. Here is what SSH said:' + print child.before, child.after + sys.exit (1) + if i == 1: # SSH does not have the public key. Just accept it. + child.sendline ('yes') + child.expect ('[Pp]assword: ') + child.sendline(password) + # Now we are either at the command prompt or + # the login process is asking for our terminal type. + i = child.expect (['Permission denied', TERMINAL_PROMPT, COMMAND_PROMPT]) + if i == 0: + print 'Permission denied on host:', host + sys.exit (1) + if i == 1: + child.sendline (TERMINAL_TYPE) + child.expect (COMMAND_PROMPT) + return child + +# (current) UNIX password: +def change_password(child, user, oldpassword, newpassword): + + child.sendline('passwd') + i = child.expect(['[Oo]ld [Pp]assword', '.current.*password', '[Nn]ew [Pp]assword']) + # Root does not require old password, so it gets to bypass the next step. + if i == 0 or i == 1: + child.sendline(oldpassword) + child.expect('[Nn]ew [Pp]assword') + child.sendline(newpassword) + i = child.expect(['[Nn]ew [Pp]assword', '[Rr]etype', '[Rr]e-enter']) + if i == 0: + print 'Host did not like new password. Here is what it said...' + print child.before + child.send (chr(3)) # Ctrl-C + child.sendline('') # This should tell remote passwd command to quit. + return + child.sendline(newpassword) + +def main(): + + if len(sys.argv) <= 1: + print USAGE + return 1 + + user = raw_input('Username: ') + password = getpass.getpass('Current Password: ') + newpassword = getpass.getpass('New Password: ') + newpasswordconfirm = getpass.getpass('Confirm New Password: ') + if newpassword != newpasswordconfirm: + print 'New Passwords do not match.' + return 1 + + for host in sys.argv[1:]: + child = login(host, user, password) + if child == None: + print 'Could not login to host:', host + continue + print 'Changing password on host:', host + change_password(child, user, password, newpassword) + child.expect(COMMAND_PROMPT) + child.sendline('exit') + +if __name__ == '__main__': + try: + main() + except pexpect.ExceptionPexpect, e: + print str(e) + diff --git a/third_party/Python/module/pexpect-2.4/examples/python.py b/third_party/Python/module/pexpect-2.4/examples/python.py new file mode 100644 index 000000000000..d8c986652e6d --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/python.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +"""This starts the python interpreter; captures the startup message; then gives +the user interactive control over the session. Why? For fun... """ + +# Don't do this unless you like being John Malkovich +# c = pexpect.spawn ('/usr/bin/env python ./python.py') + +import pexpect +c = pexpect.spawn ('/usr/bin/env python') +c.expect ('>>>') +print 'And now for something completely different...' +f = lambda s:s and f(s[1:])+s[0] # Makes a function to reverse a string. +print f(c.before) +print 'Yes, it\'s python, but it\'s backwards.' +print +print 'Escape character is \'^]\'.' +print c.after, +c.interact() +c.kill(1) +print 'is alive:', c.isalive() + diff --git a/third_party/Python/module/pexpect-2.4/examples/rippy.py b/third_party/Python/module/pexpect-2.4/examples/rippy.py new file mode 100644 index 000000000000..e89c314c323c --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/rippy.py @@ -0,0 +1,993 @@ +#!/usr/bin/env python + +"""Rippy! + +This script helps to convert video from one format to another. +This is useful for ripping DVD to mpeg4 video (XviD, DivX). + +Features: + * automatic crop detection + * mp3 audio compression with resampling options + * automatic bitrate calculation based on desired target size + * optional interlace removal, b/w video optimization, video scaling + +Run the script with no arguments to start with interactive prompts: + rippy.py +Run the script with the filename of a config to start automatic mode: + rippy.py rippy.conf + +After Rippy is finished it saves the current configuation in a file called +'rippy.conf' in the local directoy. This can be used to rerun process using the +exact same settings by passing the filename of the conf file as an argument to +Rippy. Rippy will read the options from the file instead of asking you for +options interactively. So if you run rippy with 'dry_run=1' then you can run +the process again later using the 'rippy.conf' file. Don't forget to edit +'rippy.conf' to set 'dry_run=0'! + +If you run rippy with 'dry_run' and 'verbose' true then the output generated is +valid command line commands. you could (in theory) cut-and-paste the commands +to a shell prompt. You will need to tweak some values such as crop area and bit +rate because these cannot be calculated in a dry run. This is useful if you +want to get an idea of what Rippy plans to do. + +For all the trouble that Rippy goes through to calculate the best bitrate for a +desired target video size it sometimes fails to get it right. Sometimes the +final video size will differ more than you wanted from the desired size, but if +you are really motivated and have a lot of time on your hands then you can run +Rippy again with a manually calculated bitrate. After all compression is done +the first time Rippy will recalculate the bitrate to give you the nearly exact +bitrate that would have worked. You can then edit the 'rippy.conf' file; set +the video_bitrate with this revised bitrate; and then run Rippy all over again. +There is nothing like 4-pass video compression to get it right! Actually, this +could be done in three passes since I don't need to do the second pass +compression before I calculate the revised bitrate. I'm also considering an +enhancement where Rippy would compress ten spread out chunks, 1-minute in +length to estimate the bitrate. + +Free, open source, and all that good stuff. +Rippy Copyright (c) 2006 Noah Spurrier + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. + +Noah Spurrier +$Id: rippy.py 517 2008-08-18 22:23:56Z noah $ +""" + +import sys, os, re, math, stat, getopt, traceback, types, time +import pexpect + +__version__ = '1.2' +__revision__ = '$Revision: 11 $' +__all__ = ['main', __version__, __revision__] + +GLOBAL_LOGFILE_NAME = "rippy_%d.log" % os.getpid() +GLOBAL_LOGFILE = open (GLOBAL_LOGFILE_NAME, "wb") + +############################################################################### +# This giant section defines the prompts and defaults used in interactive mode. +############################################################################### +# Python dictionaries are unordered, so +# I have this list that maintains the order of the keys. +prompts_key_order = ( +'verbose_flag', +'dry_run_flag', +'video_source_filename', +'video_chapter', +'video_final_filename', +'video_length', +'video_aspect_ratio', +'video_scale', +'video_encode_passes', +'video_codec', +'video_fourcc_override', +'video_bitrate', +'video_bitrate_overhead', +'video_target_size', +'video_deinterlace_flag', +'video_crop_area', +'video_gray_flag', +'subtitle_id', +'audio_id', +'audio_codec', +'audio_raw_filename', +'audio_volume_boost', +'audio_sample_rate', +'audio_bitrate', +#'audio_lowpass_filter', +'delete_tmp_files_flag' +) +# +# The 'prompts' dictionary holds all the messages shown to the user in +# interactive mode. The 'prompts' dictionary schema is defined as follows: +# prompt_key : ( default value, prompt string, help string, level of difficulty (0,1,2) ) +# +prompts = { +'video_source_filename':("dvd://1", 'video source filename?', """This is the filename of the video that you want to convert from. +It can be any file that mencoder supports. +You can also choose a DVD device using the dvd://1 syntax. +Title 1 is usually the main title on a DVD.""",0), +'video_chapter':("none",'video chapter?',"""This is the chapter number. Usually disks such as TV series seasons will be divided into chapters. Maybe be set to none.""",0), +'video_final_filename':("video_final.avi", "video final filename?", """This is the name of the final video.""",0), +'audio_raw_filename':("audiodump.wav", "audio raw filename?", """This is the audio raw PCM filename. This is prior to compression. +Note that mplayer automatically names this audiodump.wav, so don't change this.""",1000), +#'audio_compressed_filename':("audiodump.mp3","Audio compressed filename?", """This is the name of the compressed audio that will be mixed +#into the final video. Normally you don't need to change this.""",2), +'video_length':("none","video length in seconds?","""This sets the length of the video in seconds. This is used to estimate the +bitrate for a target video file size. Set to 'calc' to have Rippy calculate +the length. Set to 'none' if you don't want rippy to estimate the bitrate -- +you will have to manually specify bitrate.""",1), +'video_aspect_ratio':("calc","aspect ratio?","""This sets the aspect ratio of the video. Most DVDs are 16/9 or 4/3.""",1), +'video_scale':("none","video scale?","""This scales the video to the given output size. The default is to do no scaling. +You may type in a resolution such as 320x240 or you may use presets. + qntsc: 352x240 (NTSC quarter screen) + qpal: 352x288 (PAL quarter screen) + ntsc: 720x480 (standard NTSC) + pal: 720x576 (standard PAL) + sntsc: 640x480 (square pixel NTSC) + spal: 768x576 (square pixel PAL)""",1), +'video_codec':("mpeg4","video codec?","""This is the video compression to use. This is passed directly to mencoder, so +any format that it recognizes should work. For XviD or DivX use mpeg4. +Almost all MS Windows systems support wmv2 out of the box. +Some common codecs include: +mjpeg, h263, h263p, h264, mpeg4, msmpeg4, wmv1, wmv2, mpeg1video, mpeg2video, huffyuv, ffv1. +""",2), +'audio_codec':("mp3","audio codec?","""This is the audio compression to use. This is passed directly to mencoder, so +any format that it recognizes will work. +Some common codecs include: +mp3, mp2, aac, pcm +See mencoder manual for details.""",2), +'video_fourcc_override':("XVID","force fourcc code?","""This forces the fourcc codec to the given value. XVID is safest for Windows. +The following are common fourcc values: + FMP4 - This is the mencoder default. This is the "real" value. + XVID - used by Xvid (safest) + DX50 - + MP4S - Microsoft""",2), +'video_encode_passes':("1","number of encode passes?","""This sets how many passes to use to encode the video. You can choose 1 or 2. +Using two pases takes twice as long as one pass, but produces a better +quality video. I found that the improvement is not that impressive.""",1), +'verbose_flag':("Y","verbose output?","""This sets verbose output. If true then all commands and arguments are printed +before they are run. This is useful to see exactly how commands are run.""",1), +'dry_run_flag':("N","dry run?","""This sets 'dry run' mode. If true then commands are not run. This is useful +if you want to see what would the script would do.""",1), +'video_bitrate':("calc","video bitrate?","""This sets the video bitrate. This overrides video_target_size. +Set to 'calc' to automatically estimate the bitrate based on the +video final target size. If you set video_length to 'none' then +you will have to specify this video_bitrate.""",1), +'video_target_size':("737280000","video final target size?","""This sets the target video size that you want to end up with. +This is over-ridden by video_bitrate. In other words, if you specify +video_bitrate then video_target_size is ignored. +Due to the unpredictable nature of VBR compression the final video size +may not exactly match. The following are common CDR sizes: + 180MB CDR (21 minutes) holds 193536000 bytes + 550MB CDR (63 minutes) holds 580608000 bytes + 650MB CDR (74 minutes) holds 681984000 bytes + 700MB CDR (80 minutes) holds 737280000 bytes""",0), +'video_bitrate_overhead':("1.0","bitrate overhead factor?","""Adjust this value if you want to leave more room for +other files such as subtitle files. +If you specify video_bitrate then this value is ignored.""",2), +'video_crop_area':("detect","crop area?","""This sets the crop area to remove black bars from the top or sides of the video. +This helps save space. Set to 'detect' to automatically detect the crop area. +Set to 'none' to not crop the video. Normally you don't need to change this.""",1), +'video_deinterlace_flag':("N","is the video interlaced?","""This sets the deinterlace flag. If set then mencoder will be instructed +to filter out interlace artifacts (using '-vf pp=md').""",1), +'video_gray_flag':("N","is the video black and white (gray)?","""This improves output for black and white video.""",1), +'subtitle_id':("None","Subtitle ID stream?","""This selects the subtitle stream to extract from the source video. +Normally, 0 is the English subtitle stream for a DVD. +Subtitles IDs with higher numbers may be other languages.""",1), +'audio_id':("128","audio ID stream?","""This selects the audio stream to extract from the source video. +If your source is a VOB file (DVD) then stream IDs start at 128. +Normally, 128 is the main audio track for a DVD. +Tracks with higher numbers may be other language dubs or audio commentary.""",1), +'audio_sample_rate':("32000","audio sample rate (Hz) 48000, 44100, 32000, 24000, 12000","""This sets the rate at which the compressed audio will be resampled. +DVD audio is 48 kHz whereas music CDs use 44.1 kHz. The higher the sample rate +the more space the audio track will take. That will leave less space for video. +32 kHz is a good trade-off if you are trying to fit a video onto a CD.""",1), +'audio_bitrate':("96","audio bitrate (kbit/s) 192, 128, 96, 64?","""This sets the bitrate for MP3 audio compression. +The higher the bitrate the more space the audio track will take. +That will leave less space for video. Most people find music to be acceptable +at 128 kBitS. 96 kBitS is a good trade-off if you are trying to fit a video onto a CD.""",1), +'audio_volume_boost':("none","volume dB boost?","""Many DVDs have very low audio volume. This sets an audio volume boost in Decibels. +Values of 6 to 10 usually adjust quiet DVDs to a comfortable level.""",1), +#'audio_lowpass_filter':("16","audio lowpass filter (kHz)?","""This sets the low-pass filter for the audio. +#Normally this should be half of the audio sample rate. +#This improves audio compression and quality. +#Normally you don't need to change this.""",1), +'delete_tmp_files_flag':("N","delete temporary files when finished?","""If Y then %s, audio_raw_filename, and 'divx2pass.log' will be deleted at the end."""%GLOBAL_LOGFILE_NAME,1) +} + +############################################################################## +# This is the important convert control function +############################################################################## +def convert (options): + """This is the heart of it all -- this performs an end-to-end conversion of + a video from one format to another. It requires a dictionary of options. + The conversion process will also add some keys to the dictionary + such as length of the video and crop area. The dictionary is returned. + This options dictionary could be used again to repeat the convert process + (it is also saved to rippy.conf as text). + """ + if options['subtitle_id'] is not None: + print "# extract subtitles" + apply_smart (extract_subtitles, options) + else: + print "# do not extract subtitles." + + # Optimization + # I really only need to calculate the exact video length if the user + # selected 'calc' for video_bitrate + # or + # selected 'detect' for video_crop_area. + if options['video_bitrate']=='calc' or options['video_crop_area']=='detect': + # As strange as it seems, the only reliable way to calculate the length + # of a video (in seconds) is to extract the raw, uncompressed PCM audio stream + # and then calculate the length of that. This is because MP4 video is VBR, so + # you cannot get exact time based on compressed size. + if options['video_length']=='calc': + print "# extract PCM raw audio to %s" % (options['audio_raw_filename']) + apply_smart (extract_audio, options) + options['video_length'] = apply_smart (get_length, options) + print "# Length of raw audio file : %d seconds (%0.2f minutes)" % (options['video_length'], float(options['video_length'])/60.0) + if options['video_bitrate']=='calc': + options['video_bitrate'] = options['video_bitrate_overhead'] * apply_smart (calc_video_bitrate, options) + print "# video bitrate : " + str(options['video_bitrate']) + if options['video_crop_area']=='detect': + options['video_crop_area'] = apply_smart (crop_detect, options) + print "# crop area : " + str(options['video_crop_area']) + print "# compression estimate" + print apply_smart (compression_estimate, options) + + print "# compress video" + apply_smart (compress_video, options) + 'audio_volume_boost', + + print "# delete temporary files:", + if options['delete_tmp_files_flag']: + print "yes" + apply_smart (delete_tmp_files, options) + else: + print "no" + + # Finish by saving options to rippy.conf and + # calclating if final_size is less than target_size. + o = ["# options used to create video\n"] + video_actual_size = get_filesize (options['video_final_filename']) + if options['video_target_size'] != 'none': + revised_bitrate = calculate_revised_bitrate (options['video_bitrate'], options['video_target_size'], video_actual_size) + o.append("# revised video_bitrate : %d\n" % revised_bitrate) + for k,v in options.iteritems(): + o.append (" %30s : %s\n" % (k, v)) + print '# '.join(o) + fout = open("rippy.conf","wb").write(''.join(o)) + print "# final actual video size = %d" % video_actual_size + if options['video_target_size'] != 'none': + if video_actual_size > options['video_target_size']: + print "# FINAL VIDEO SIZE IS GREATER THAN DESIRED TARGET" + print "# final video size is %d bytes over target size" % (video_actual_size - options['video_target_size']) + else: + print "# final video size is %d bytes under target size" % (options['video_target_size'] - video_actual_size) + print "# If you want to run the entire compression process all over again" + print "# to get closer to the target video size then trying using a revised" + print "# video_bitrate of %d" % revised_bitrate + + return options + +############################################################################## + +def exit_with_usage(exit_code=1): + print globals()['__doc__'] + print 'version:', globals()['__version__'] + sys.stdout.flush() + os._exit(exit_code) + +def check_missing_requirements (): + """This list of missing requirements (mencoder, mplayer, lame, and mkvmerge). + Returns None if all requirements are in the execution path. + """ + missing = [] + if pexpect.which("mencoder") is None: + missing.append("mencoder") + if pexpect.which("mplayer") is None: + missing.append("mplayer") + cmd = "mencoder -oac help" + (command_output, exitstatus) = run(cmd) + ar = re.findall("(mp3lame)", command_output) + if len(ar)==0: + missing.append("Mencoder was not compiled with mp3lame support.") + + #if pexpect.which("lame") is None: + # missing.append("lame") + #if pexpect.which("mkvmerge") is None: + # missing.append("mkvmerge") + if len(missing)==0: + return None + return missing + +def input_option (message, default_value="", help=None, level=0, max_level=0): + """This is a fancy raw_input function. + If the user enters '?' then the contents of help is printed. + + The 'level' and 'max_level' are used to adjust which advanced options + are printed. 'max_level' is the level of options that the user wants + to see. 'level' is the level of difficulty for this particular option. + If this level is <= the max_level the user wants then the + message is printed and user input is allowed; otherwise, the + default value is returned automatically without user input. + """ + if default_value != '': + message = "%s [%s] " % (message, default_value) + if level > max_level: + return default_value + while 1: + user_input = raw_input (message) + if user_input=='?': + print help + elif user_input=='': + return default_value + else: + break + return user_input + +def progress_callback (d=None): + """This callback simply prints a dot to show activity. + This is used when running external commands with pexpect.run. + """ + sys.stdout.write (".") + sys.stdout.flush() + +def run(cmd): + global GLOBAL_LOGFILE + print >>GLOBAL_LOGFILE, cmd + (command_output, exitstatus) = pexpect.run(cmd, events={pexpect.TIMEOUT:progress_callback}, timeout=5, withexitstatus=True, logfile=GLOBAL_LOGFILE) + if exitstatus != 0: + print "RUN FAILED. RETURNED EXIT STATUS:", exitstatus + print >>GLOBAL_LOGFILE, "RUN FAILED. RETURNED EXIT STATUS:", exitstatus + return (command_output, exitstatus) + +def apply_smart (func, args): + """This is similar to func(**args), but this won't complain about + extra keys in 'args'. This ignores keys in 'args' that are + not required by 'func'. This passes None to arguments that are + not defined in 'args'. That's fine for arguments with a default valeue, but + that's a bug for required arguments. I should probably raise a TypeError. + The func parameter can be a function reference or a string. + If it is a string then it is converted to a function reference. + """ + if type(func) is type(''): + if func in globals(): + func = globals()[func] + else: + raise NameError("name '%s' is not defined" % func) + if hasattr(func,'im_func'): # Handle case when func is a class method. + func = func.im_func + argcount = func.func_code.co_argcount + required_args = dict([(k,args.get(k)) for k in func.func_code.co_varnames[:argcount]]) + return func(**required_args) + +def count_unique (items): + """This takes a list and returns a sorted list of tuples with a count of each unique item in the list. + Example 1: + count_unique(['a','b','c','a','c','c','a','c','c']) + returns: + [(5,'c'), (3,'a'), (1,'b')] + Example 2 -- get the most frequent item in a list: + count_unique(['a','b','c','a','c','c','a','c','c'])[0][1] + returns: + 'c' + """ + stats = {} + for i in items: + if i in stats: + stats[i] = stats[i] + 1 + else: + stats[i] = 1 + stats = [(v, k) for k, v in stats.items()] + stats.sort() + stats.reverse() + return stats + +def calculate_revised_bitrate (video_bitrate, video_target_size, video_actual_size): + """This calculates a revised video bitrate given the video_bitrate used, + the actual size that resulted, and the video_target_size. + This can be used if you want to compress the video all over again in an + attempt to get closer to the video_target_size. + """ + return int(math.floor(video_bitrate * (float(video_target_size) / float(video_actual_size)))) + +def get_aspect_ratio (video_source_filename): + """This returns the aspect ratio of the original video. + This is usualy 1.78:1(16/9) or 1.33:1(4/3). + This function is very lenient. It basically guesses 16/9 whenever + it cannot figure out the aspect ratio. + """ + cmd = "mplayer '%s' -vo png -ao null -frames 1" % video_source_filename + (command_output, exitstatus) = run(cmd) + ar = re.findall("Movie-Aspect is ([0-9]+\.?[0-9]*:[0-9]+\.?[0-9]*)", command_output) + if len(ar)==0: + return '16/9' + if ar[0] == '1.78:1': + return '16/9' + if ar[0] == '1.33:1': + return '4/3' + return '16/9' + #idh = re.findall("ID_VIDEO_HEIGHT=([0-9]+)", command_output) + #if len(idw)==0 or len(idh)==0: + # print 'WARNING!' + # print 'Could not get aspect ration. Assuming 1.78:1 (16/9).' + # return 1.78 + #return float(idw[0])/float(idh[0]) +#ID_VIDEO_WIDTH=720 +#ID_VIDEO_HEIGHT=480 +#Movie-Aspect is 1.78:1 - prescaling to correct movie aspect. + + +def get_aid_list (video_source_filename): + """This returns a list of audio ids in the source video file. + TODO: Also extract ID_AID_nnn_LANG to associate language. Not all DVDs include this. + """ + cmd = "mplayer '%s' -vo null -ao null -frames 0 -identify" % video_source_filename + (command_output, exitstatus) = run(cmd) + idl = re.findall("ID_AUDIO_ID=([0-9]+)", command_output) + idl.sort() + return idl + +def get_sid_list (video_source_filename): + """This returns a list of subtitle ids in the source video file. + TODO: Also extract ID_SID_nnn_LANG to associate language. Not all DVDs include this. + """ + cmd = "mplayer '%s' -vo null -ao null -frames 0 -identify" % video_source_filename + (command_output, exitstatus) = run(cmd) + idl = re.findall("ID_SUBTITLE_ID=([0-9]+)", command_output) + idl.sort() + return idl + +def extract_audio (video_source_filename, audio_id=128, verbose_flag=0, dry_run_flag=0): + """This extracts the given audio_id track as raw uncompressed PCM from the given source video. + Note that mplayer always saves this to audiodump.wav. + At this time there is no way to set the output audio name. + """ + #cmd = "mplayer %(video_source_filename)s -vc null -vo null -aid %(audio_id)s -ao pcm:fast -noframedrop" % locals() + cmd = "mplayer -quiet '%(video_source_filename)s' -vc dummy -vo null -aid %(audio_id)s -ao pcm:fast -noframedrop" % locals() + if verbose_flag: print cmd + if not dry_run_flag: + run(cmd) + print + +def extract_subtitles (video_source_filename, subtitle_id=0, verbose_flag=0, dry_run_flag=0): + """This extracts the given subtitle_id track as VOBSUB format from the given source video. + """ + cmd = "mencoder -quiet '%(video_source_filename)s' -o /dev/null -nosound -ovc copy -vobsubout subtitles -vobsuboutindex 0 -sid %(subtitle_id)s" % locals() + if verbose_flag: print cmd + if not dry_run_flag: + run(cmd) + print + +def get_length (audio_raw_filename): + """This attempts to get the length of the media file (length is time in seconds). + This should not be confused with size (in bytes) of the file data. + This is best used on a raw PCM AUDIO file because mplayer cannot get an accurate + time for many compressed video and audio formats -- notably MPEG4 and MP3. + Weird... + This returns -1 if it cannot get the length of the given file. + """ + cmd = "mplayer %s -vo null -ao null -frames 0 -identify" % audio_raw_filename + (command_output, exitstatus) = run(cmd) + idl = re.findall("ID_LENGTH=([0-9.]*)", command_output) + idl.sort() + if len(idl) != 1: + print "ERROR: cannot get length of raw audio file." + print "command_output of mplayer identify:" + print command_output + print "parsed command_output:" + print str(idl) + return -1 + return float(idl[0]) + +def get_filesize (filename): + """This returns the number of bytes a file takes on storage.""" + return os.stat(filename)[stat.ST_SIZE] + +def calc_video_bitrate (video_target_size, audio_bitrate, video_length, extra_space=0, dry_run_flag=0): + """This gives an estimate of the video bitrate necessary to + fit the final target size. This will take into account room to + fit the audio and extra space if given (for container overhead or whatnot). + video_target_size is in bytes, + audio_bitrate is bits per second (96, 128, 256, etc.) ASSUMING CBR, + video_length is in seconds, + extra_space is in bytes. + a 180MB CDR (21 minutes) holds 193536000 bytes. + a 550MB CDR (63 minutes) holds 580608000 bytes. + a 650MB CDR (74 minutes) holds 681984000 bytes. + a 700MB CDR (80 minutes) holds 737280000 bytes. + """ + if dry_run_flag: + return -1 + if extra_space is None: extra_space = 0 + #audio_size = os.stat(audio_compressed_filename)[stat.ST_SIZE] + audio_size = (audio_bitrate * video_length * 1000) / 8.0 + video_target_size = video_target_size - audio_size - extra_space + return (int)(calc_video_kbitrate (video_target_size, video_length)) + +def calc_video_kbitrate (target_size, length_secs): + """Given a target byte size free for video data, this returns the bitrate in kBit/S. + For mencoder vbitrate 1 kBit = 1000 Bits -- not 1024 bits. + target_size = bitrate * 1000 * length_secs / 8 + target_size = bitrate * 125 * length_secs + bitrate = target_size/(125*length_secs) + """ + return int(target_size / (125.0 * length_secs)) + +def crop_detect (video_source_filename, video_length, dry_run_flag=0): + """This attempts to figure out the best crop for the given video file. + Basically it runs crop detect for 10 seconds on five different places in the video. + It picks the crop area that was most often detected. + """ + skip = int(video_length/9) # offset to skip (-ss option in mencoder) + sample_length = 10 + cmd1 = "mencoder '%s' -quiet -ss %d -endpos %d -o /dev/null -nosound -ovc lavc -vf cropdetect" % (video_source_filename, skip, sample_length) + cmd2 = "mencoder '%s' -quiet -ss %d -endpos %d -o /dev/null -nosound -ovc lavc -vf cropdetect" % (video_source_filename, 2*skip, sample_length) + cmd3 = "mencoder '%s' -quiet -ss %d -endpos %d -o /dev/null -nosound -ovc lavc -vf cropdetect" % (video_source_filename, 4*skip, sample_length) + cmd4 = "mencoder '%s' -quiet -ss %d -endpos %d -o /dev/null -nosound -ovc lavc -vf cropdetect" % (video_source_filename, 6*skip, sample_length) + cmd5 = "mencoder '%s' -quiet -ss %d -endpos %d -o /dev/null -nosound -ovc lavc -vf cropdetect" % (video_source_filename, 8*skip, sample_length) + if dry_run_flag: + return "0:0:0:0" + (command_output1, exitstatus1) = run(cmd1) + (command_output2, exitstatus2) = run(cmd2) + (command_output3, exitstatus3) = run(cmd3) + (command_output4, exitstatus4) = run(cmd4) + (command_output5, exitstatus5) = run(cmd5) + idl = re.findall("-vf crop=([0-9]+:[0-9]+:[0-9]+:[0-9]+)", command_output1) + idl = idl + re.findall("-vf crop=([0-9]+:[0-9]+:[0-9]+:[0-9]+)", command_output2) + idl = idl + re.findall("-vf crop=([0-9]+:[0-9]+:[0-9]+:[0-9]+)", command_output3) + idl = idl + re.findall("-vf crop=([0-9]+:[0-9]+:[0-9]+:[0-9]+)", command_output4) + idl = idl + re.findall("-vf crop=([0-9]+:[0-9]+:[0-9]+:[0-9]+)", command_output5) + items_count = count_unique(idl) + return items_count[0][1] + + +def build_compression_command (video_source_filename, video_final_filename, video_target_size, audio_id=128, video_bitrate=1000, video_codec='mpeg4', audio_codec='mp3', video_fourcc_override='FMP4', video_gray_flag=0, video_crop_area=None, video_aspect_ratio='16/9', video_scale=None, video_encode_passes=2, video_deinterlace_flag=0, audio_volume_boost=None, audio_sample_rate=None, audio_bitrate=None, seek_skip=None, seek_length=None, video_chapter=None): +#Notes:For DVD, VCD, and SVCD use acodec=mp2 and vcodec=mpeg2video: +#mencoder movie.avi -o movie.VOB -ovc lavc -oac lavc -lavcopts acodec=mp2:abitrate=224:vcodec=mpeg2video:vbitrate=2000 + + # + # build video filter (-vf) argument + # + video_filter = '' + if video_crop_area and video_crop_area.lower()!='none': + video_filter = video_filter + 'crop=%s' % video_crop_area + if video_deinterlace_flag: + if video_filter != '': + video_filter = video_filter + ',' + video_filter = video_filter + 'pp=md' + if video_scale and video_scale.lower()!='none': + if video_filter != '': + video_filter = video_filter + ',' + video_filter = video_filter + 'scale=%s' % video_scale + # optional video rotation -- were you holding your camera sideways? + #if video_filter != '': + # video_filter = video_filter + ',' + #video_filter = video_filter + 'rotate=2' + if video_filter != '': + video_filter = '-vf ' + video_filter + + # + # build chapter argument + # + if video_chapter is not None: + chapter = '-chapter %d-%d' %(video_chapter,video_chapter) + else: + chapter = '' +# chapter = '-chapter 2-2' + + # + # build audio_filter argument + # + audio_filter = '' + if audio_sample_rate: + if audio_filter != '': + audio_filter = audio_filter + ',' + audio_filter = audio_filter + 'lavcresample=%s' % audio_sample_rate + if audio_volume_boost is not None: + if audio_filter != '': + audio_filter = audio_filter + ',' + audio_filter = audio_filter + 'volume=%0.1f:1'%audio_volume_boost + if audio_filter != '': + audio_filter = '-af ' + audio_filter + # + #if audio_sample_rate: + # audio_filter = ('-srate %d ' % audio_sample_rate) + audio_filter + + # + # build lavcopts argument + # + #lavcopts = '-lavcopts vcodec=%s:vbitrate=%d:mbd=2:aspect=%s:acodec=%s:abitrate=%d:vpass=1' % (video_codec,video_bitrate,audio_codec,audio_bitrate) + lavcopts = '-lavcopts vcodec=%(video_codec)s:vbitrate=%(video_bitrate)d:mbd=2:aspect=%(video_aspect_ratio)s:acodec=%(audio_codec)s:abitrate=%(audio_bitrate)d:vpass=1' % (locals()) + if video_gray_flag: + lavcopts = lavcopts + ':gray' + + seek_filter = '' + if seek_skip is not None: + seek_filter = '-ss %s' % (str(seek_skip)) + if seek_length is not None: + seek_filter = seek_filter + ' -endpos %s' % (str(seek_length)) + +# cmd = "mencoder -quiet -info comment='Arkivist' '%(video_source_filename)s' %(seek_filter)s %(chapter)s -aid %(audio_id)s -o '%(video_final_filename)s' -ffourcc %(video_fourcc_override)s -ovc lavc -oac lavc %(lavcopts)s %(video_filter)s %(audio_filter)s" % locals() + cmd = "mencoder -quiet -info comment='Arkivist' '%(video_source_filename)s' %(seek_filter)s %(chapter)s -aid %(audio_id)s -o '%(video_final_filename)s' -ffourcc %(video_fourcc_override)s -ovc lavc -oac mp3lame %(lavcopts)s %(video_filter)s %(audio_filter)s" % locals() + return cmd + +def compression_estimate (video_length, video_source_filename, video_final_filename, video_target_size, audio_id=128, video_bitrate=1000, video_codec='mpeg4', audio_codec='mp3', video_fourcc_override='FMP4', video_gray_flag=0, video_crop_area=None, video_aspect_ratio='16/9', video_scale=None, video_encode_passes=2, video_deinterlace_flag=0, audio_volume_boost=None, audio_sample_rate=None, audio_bitrate=None): + """This attempts to figure out the best compression ratio for a given set of compression options. + """ + # TODO Need to account for AVI overhead. + skip = int(video_length/9) # offset to skip (-ss option in mencoder) + sample_length = 10 + cmd1 = build_compression_command (video_source_filename, "compression_test_1.avi", video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, skip, sample_length) + cmd2 = build_compression_command (video_source_filename, "compression_test_2.avi", video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, skip*2, sample_length) + cmd3 = build_compression_command (video_source_filename, "compression_test_3.avi", video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, skip*4, sample_length) + cmd4 = build_compression_command (video_source_filename, "compression_test_4.avi", video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, skip*6, sample_length) + cmd5 = build_compression_command (video_source_filename, "compression_test_5.avi", video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, skip*8, sample_length) + run(cmd1) + run(cmd2) + run(cmd3) + run(cmd4) + run(cmd5) + size = get_filesize ("compression_test_1.avi")+get_filesize ("compression_test_2.avi")+get_filesize ("compression_test_3.avi")+get_filesize ("compression_test_4.avi")+get_filesize ("compression_test_5.avi") + return (size / 5.0) + +def compress_video (video_source_filename, video_final_filename, video_target_size, audio_id=128, video_bitrate=1000, video_codec='mpeg4', audio_codec='mp3', video_fourcc_override='FMP4', video_gray_flag=0, video_crop_area=None, video_aspect_ratio='16/9', video_scale=None, video_encode_passes=2, video_deinterlace_flag=0, audio_volume_boost=None, audio_sample_rate=None, audio_bitrate=None, seek_skip=None, seek_length=None, video_chapter=None, verbose_flag=0, dry_run_flag=0): + """This compresses the video and audio of the given source video filename to the transcoded filename. + This does a two-pass compression (I'm assuming mpeg4, I should probably make this smarter for other formats). + """ + # + # do the first pass video compression + # + #cmd = "mencoder -quiet '%(video_source_filename)s' -ss 65 -endpos 20 -aid %(audio_id)s -o '%(video_final_filename)s' -ffourcc %(video_fourcc_override)s -ovc lavc -oac lavc %(lavcopts)s %(video_filter)s %(audio_filter)s" % locals() + + cmd = build_compression_command (video_source_filename, video_final_filename, video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, seek_skip, seek_length, video_chapter) + if verbose_flag: print cmd + if not dry_run_flag: + run(cmd) + print + + # If not doing two passes then return early. + if video_encode_passes!='2': + return + + if verbose_flag: + video_actual_size = get_filesize (video_final_filename) + if video_actual_size > video_target_size: + print "=======================================================" + print "WARNING!" + print "First pass compression resulted in" + print "actual file size greater than target size." + print "Second pass will be too big." + print "=======================================================" + + # + # do the second pass video compression + # + cmd = cmd.replace ('vpass=1', 'vpass=2') + if verbose_flag: print cmd + if not dry_run_flag: + run(cmd) + print + return + +def compress_audio (audio_raw_filename, audio_compressed_filename, audio_lowpass_filter=None, audio_sample_rate=None, audio_bitrate=None, verbose_flag=0, dry_run_flag=0): + """This is depricated. + This compresses the raw audio file to the compressed audio filename. + """ + cmd = 'lame -h --athaa-sensitivity 1' # --cwlimit 11" + if audio_lowpass_filter: + cmd = cmd + ' --lowpass ' + audio_lowpass_filter + if audio_bitrate: + #cmd = cmd + ' --abr ' + audio_bitrate + cmd = cmd + ' --cbr -b ' + audio_bitrate + if audio_sample_rate: + cmd = cmd + ' --resample ' + audio_sample_rate + cmd = cmd + ' ' + audio_raw_filename + ' ' + audio_compressed_filename + if verbose_flag: print cmd + if not dry_run_flag: + (command_output, exitstatus) = run(cmd) + print + if exitstatus != 0: + raise Exception('ERROR: lame failed to compress raw audio file.') + +def mux (video_final_filename, video_transcoded_filename, audio_compressed_filename, video_container_format, verbose_flag=0, dry_run_flag=0): + """This is depricated. I used to use a three-pass encoding where I would mix the audio track separately, but + this never worked very well (loss of audio sync).""" + if video_container_format.lower() == 'mkv': # Matroska + mux_mkv (video_final_filename, video_transcoded_filename, audio_compressed_filename, verbose_flag, dry_run_flag) + if video_container_format.lower() == 'avi': + mux_avi (video_final_filename, video_transcoded_filename, audio_compressed_filename, verbose_flag, dry_run_flag) + +def mux_mkv (video_final_filename, video_transcoded_filename, audio_compressed_filename, verbose_flag=0, dry_run_flag=0): + """This is depricated.""" + cmd = 'mkvmerge -o %s --noaudio %s %s' % (video_final_filename, video_transcoded_filename, audio_compressed_filename) + if verbose_flag: print cmd + if not dry_run_flag: + run(cmd) + print + +def mux_avi (video_final_filename, video_transcoded_filename, audio_compressed_filename, verbose_flag=0, dry_run_flag=0): + """This is depricated.""" + pass +# cmd = "mencoder -quiet -oac copy -ovc copy -o '%s' -audiofile %s '%s'" % (video_final_filename, audio_compressed_filename, video_transcoded_filename) +# if verbose_flag: print cmd +# if not dry_run_flag: +# run(cmd) +# print + +def delete_tmp_files (audio_raw_filename, verbose_flag=0, dry_run_flag=0): + global GLOBAL_LOGFILE_NAME + file_list = ' '.join([GLOBAL_LOGFILE_NAME, 'divx2pass.log', audio_raw_filename ]) + cmd = 'rm -f ' + file_list + if verbose_flag: print cmd + if not dry_run_flag: + run(cmd) + print + +############################################################################## +# This is the interactive Q&A that is used if a conf file was not given. +############################################################################## +def interactive_convert (): + + global prompts, prompts_key_order + + print globals()['__doc__'] + print + print "==============================================" + print " Enter '?' at any question to get extra help." + print "==============================================" + print + + # Ask for the level of options the user wants. + # A lot of code just to print a string! + level_sort = {0:'', 1:'', 2:''} + for k in prompts: + level = prompts[k][3] + if level < 0 or level > 2: + continue + level_sort[level] += " " + prompts[k][1] + "\n" + level_sort_string = "This sets the level for advanced options prompts. Set 0 for simple, 1 for advanced, or 2 for expert.\n" + level_sort_string += "[0] Basic options:\n" + str(level_sort[0]) + "\n" + level_sort_string += "[1] Advanced options:\n" + str(level_sort[1]) + "\n" + level_sort_string += "[2] Expert options:\n" + str(level_sort[2]) + c = input_option("Prompt level (0, 1, or 2)?", "1", level_sort_string) + max_prompt_level = int(c) + + options = {} + for k in prompts_key_order: + if k == 'video_aspect_ratio': + guess_aspect = get_aspect_ratio(options['video_source_filename']) + options[k] = input_option (prompts[k][1], guess_aspect, prompts[k][2], prompts[k][3], max_prompt_level) + elif k == 'audio_id': + aid_list = get_aid_list (options['video_source_filename']) + default_id = '128' + if max_prompt_level>=prompts[k][3]: + if len(aid_list) > 1: + print "This video has more than one audio stream. The following stream audio IDs were found:" + for aid in aid_list: + print " " + aid + default_id = aid_list[0] + else: + print "WARNING!" + print "Rippy was unable to get the list of audio streams from this video." + print "If reading directly from a DVD then the DVD device might be busy." + print "Using a default setting of stream id 128 (main audio on most DVDs)." + default_id = '128' + options[k] = input_option (prompts[k][1], default_id, prompts[k][2], prompts[k][3], max_prompt_level) + elif k == 'subtitle_id': + sid_list = get_sid_list (options['video_source_filename']) + default_id = 'None' + if max_prompt_level>=prompts[k][3]: + if len(sid_list) > 0: + print "This video has one or more subtitle streams. The following stream subtitle IDs were found:" + for sid in sid_list: + print " " + sid + #default_id = sid_list[0] + default_id = prompts[k][0] + else: + print "WARNING!" + print "Unable to get the list of subtitle streams from this video. It may have none." + print "Setting default to None." + default_id = 'None' + options[k] = input_option (prompts[k][1], default_id, prompts[k][2], prompts[k][3], max_prompt_level) + elif k == 'audio_lowpass_filter': + lowpass_default = "%.1f" % (math.floor(float(options['audio_sample_rate']) / 2.0)) + options[k] = input_option (prompts[k][1], lowpass_default, prompts[k][2], prompts[k][3], max_prompt_level) + elif k == 'video_bitrate': + if options['video_length'].lower() == 'none': + options[k] = input_option (prompts[k][1], '1000', prompts[k][2], prompts[k][3], max_prompt_level) + else: + options[k] = input_option (prompts[k][1], prompts[k][0], prompts[k][2], prompts[k][3], max_prompt_level) + else: + # don't bother asking for video_target_size or video_bitrate_overhead if video_bitrate was set + if (k=='video_target_size' or k=='video_bitrate_overhead') and options['video_bitrate']!='calc': + continue + # don't bother with crop area if video length is none + if k == 'video_crop_area' and options['video_length'].lower() == 'none': + options['video_crop_area'] = 'none' + continue + options[k] = input_option (prompts[k][1], prompts[k][0], prompts[k][2], prompts[k][3], max_prompt_level) + + #options['video_final_filename'] = options['video_final_filename'] + "." + options['video_container_format'] + + print "==========================================================================" + print "Ready to Rippy!" + print + print "The following options will be used:" + for k,v in options.iteritems(): + print "%27s : %s" % (k, v) + + print + c = input_option("Continue?", "Y") + c = c.strip().lower() + if c[0] != 'y': + print "Exiting..." + os._exit(1) + return options + +def clean_options (d): + """This validates and cleans up the options dictionary. + After reading options interactively or from a conf file + we need to make sure that the values make sense and are + converted to the correct type. + 1. Any key with "_flag" in it becomes a boolean True or False. + 2. Values are normalized ("No", "None", "none" all become "none"; + "Calcluate", "c", "CALC" all become "calc"). + 3. Certain values are converted from string to int. + 4. Certain combinations of options are invalid or override each other. + This is a rather annoying function, but then so it most cleanup work. + """ + for k in d: + d[k] = d[k].strip() + # convert all flag options to 0 or 1 + if '_flag' in k: + if type(d[k]) is types.StringType: + if d[k].strip().lower()[0] in 'yt1': #Yes, True, 1 + d[k] = 1 + else: + d[k] = 0 + d['video_bitrate'] = d['video_bitrate'].lower() + if d['video_bitrate'][0]=='c': + d['video_bitrate']='calc' + else: + d['video_bitrate'] = int(float(d['video_bitrate'])) + try: + d['video_target_size'] = int(d['video_target_size']) + # shorthand magic numbers get automatically expanded + if d['video_target_size'] == 180: + d['video_target_size'] = 193536000 + elif d['video_target_size'] == 550: + d['video_target_size'] = 580608000 + elif d['video_target_size'] == 650: + d['video_target_size'] = 681984000 + elif d['video_target_size'] == 700: + d['video_target_size'] = 737280000 + except: + d['video_target_size'] = 'none' + + try: + d['video_chapter'] = int(d['video_chapter']) + except: + d['video_chapter'] = None + + try: + d['subtitle_id'] = int(d['subtitle_id']) + except: + d['subtitle_id'] = None + + try: + d['video_bitrate_overhead'] = float(d['video_bitrate_overhead']) + except: + d['video_bitrate_overhead'] = -1.0 + + d['audio_bitrate'] = int(d['audio_bitrate']) + d['audio_sample_rate'] = int(d['audio_sample_rate']) + d['audio_volume_boost'] = d['audio_volume_boost'].lower() + if d['audio_volume_boost'][0]=='n': + d['audio_volume_boost'] = None + else: + d['audio_volume_boost'] = d['audio_volume_boost'].replace('db','') + d['audio_volume_boost'] = float(d['audio_volume_boost']) + +# assert (d['video_bitrate']=='calc' and d['video_target_size']!='none') +# or (d['video_bitrate']!='calc' and d['video_target_size']=='none') + + d['video_scale'] = d['video_scale'].lower() + if d['video_scale'][0]=='n': + d['video_scale']='none' + else: + al = re.findall("([0-9]+).*?([0-9]+)", d['video_scale']) + d['video_scale']=al[0][0]+':'+al[0][1] + d['video_crop_area'] = d['video_crop_area'].lower() + if d['video_crop_area'][0]=='n': + d['video_crop_area']='none' + d['video_length'] = d['video_length'].lower() + if d['video_length'][0]=='c': + d['video_length']='calc' + elif d['video_length'][0]=='n': + d['video_length']='none' + else: + d['video_length'] = int(float(d['video_length'])) + if d['video_length']==0: + d['video_length'] = 'none' + assert (not (d['video_length']=='none' and d['video_bitrate']=='calc')) + return d + +def main (): + try: + optlist, args = getopt.getopt(sys.argv[1:], 'h?', ['help','h','?']) + except Exception, e: + print str(e) + exit_with_usage() + command_line_options = dict(optlist) + # There are a million ways to cry for help. These are but a few of them. + if [elem for elem in command_line_options if elem in ['-h','--h','-?','--?','--help']]: + exit_with_usage(0) + + missing = check_missing_requirements() + if missing is not None: + print + print "==========================================================================" + print "ERROR!" + print "Some required external commands are missing." + print "please install the following packages:" + print str(missing) + print "==========================================================================" + print + c = input_option("Continue?", "Y") + c = c.strip().lower() + if c[0] != 'y': + print "Exiting..." + os._exit(1) + + if len(args) > 0: + # cute one-line string-to-dictionary parser (two-lines if you count this comment): + options = dict(re.findall('([^: \t\n]*)\s*:\s*(".*"|[^ \t\n]*)', file(args[0]).read())) + options = clean_options(options) + convert (options) + else: + options = interactive_convert () + options = clean_options(options) + convert (options) + print "# Done!" + +if __name__ == "__main__": + try: + start_time = time.time() + print time.asctime() + main() + print time.asctime() + print "TOTAL TIME IN MINUTES:", + print (time.time() - start_time) / 60.0 + except Exception, e: + tb_dump = traceback.format_exc() + print "==========================================================================" + print "ERROR -- Unexpected exception in script." + print str(e) + print str(tb_dump) + print "==========================================================================" + print >>GLOBAL_LOGFILE, "==========================================================================" + print >>GLOBAL_LOGFILE, "ERROR -- Unexpected exception in script." + print >>GLOBAL_LOGFILE, str(e) + print >>GLOBAL_LOGFILE, str(tb_dump) + print >>GLOBAL_LOGFILE, "==========================================================================" + exit_with_usage(3) + diff --git a/third_party/Python/module/pexpect-2.4/examples/script.py b/third_party/Python/module/pexpect-2.4/examples/script.py new file mode 100644 index 000000000000..908b91241a63 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/script.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +"""This spawns a sub-shell (bash) and gives the user interactive control. The +entire shell session is logged to a file called script.log. This behaves much +like the classic BSD command 'script'. + +./script.py [-a] [-c command] {logfilename} + + logfilename : This is the name of the log file. Default is script.log. + -a : Append to log file. Default is to overwrite log file. + -c : spawn command. Default is to spawn the sh shell. + +Example: + + This will start a bash shell and append to the log named my_session.log: + + ./script.py -a -c bash my_session.log + +""" + +import os, sys, time, getopt +import signal, fcntl, termios, struct +import traceback +import pexpect + +global_pexpect_instance = None # Used by signal handler + +def exit_with_usage(): + + print globals()['__doc__'] + os._exit(1) + +def main(): + + ###################################################################### + # Parse the options, arguments, get ready, etc. + ###################################################################### + try: + optlist, args = getopt.getopt(sys.argv[1:], 'h?ac:', ['help','h','?']) + except Exception, e: + print str(e) + exit_with_usage() + options = dict(optlist) + if len(args) > 1: + exit_with_usage() + + if [elem for elem in options if elem in ['-h','--h','-?','--?','--help']]: + print "Help:" + exit_with_usage() + + if len(args) == 1: + script_filename = args[0] + else: + script_filename = "script.log" + if '-a' in options: + fout = file (script_filename, "ab") + else: + fout = file (script_filename, "wb") + if '-c' in options: + command = options['-c'] + else: + command = "sh" + + # Begin log with date/time in the form CCCCyymm.hhmmss + fout.write ('# %4d%02d%02d.%02d%02d%02d \n' % time.localtime()[:-3]) + + ###################################################################### + # Start the interactive session + ###################################################################### + p = pexpect.spawn(command) + p.logfile = fout + global global_pexpect_instance + global_pexpect_instance = p + signal.signal(signal.SIGWINCH, sigwinch_passthrough) + + print "Script recording started. Type ^] (ASCII 29) to escape from the script shell." + p.interact(chr(29)) + fout.close() + return 0 + +def sigwinch_passthrough (sig, data): + + # Check for buggy platforms (see pexpect.setwinsize()). + if 'TIOCGWINSZ' in dir(termios): + TIOCGWINSZ = termios.TIOCGWINSZ + else: + TIOCGWINSZ = 1074295912 # assume + s = struct.pack ("HHHH", 0, 0, 0, 0) + a = struct.unpack ('HHHH', fcntl.ioctl(sys.stdout.fileno(), TIOCGWINSZ , s)) + global global_pexpect_instance + global_pexpect_instance.setwinsize(a[0],a[1]) + +if __name__ == "__main__": + try: + main() + except SystemExit, e: + raise e + except Exception, e: + print "ERROR" + print str(e) + traceback.print_exc() + os._exit(1) + diff --git a/third_party/Python/module/pexpect-2.4/examples/ssh_session.py b/third_party/Python/module/pexpect-2.4/examples/ssh_session.py new file mode 100644 index 000000000000..4d0e228a7e33 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/ssh_session.py @@ -0,0 +1,94 @@ +# +# Eric S. Raymond +# +# Greatly modified by Nigel W. Moriarty +# April 2003 +# +from pexpect import * +import os, sys +import getpass +import time + +class ssh_session: + + "Session with extra state including the password to be used." + + def __init__(self, user, host, password=None, verbose=0): + + self.user = user + self.host = host + self.verbose = verbose + self.password = password + self.keys = [ + 'authenticity', + 'assword:', + '@@@@@@@@@@@@', + 'Command not found.', + EOF, + ] + + self.f = open('ssh.out','w') + + def __repr__(self): + + outl = 'class :'+self.__class__.__name__ + for attr in self.__dict__: + if attr == 'password': + outl += '\n\t'+attr+' : '+'*'*len(self.password) + else: + outl += '\n\t'+attr+' : '+str(getattr(self, attr)) + return outl + + def __exec(self, command): + + "Execute a command on the remote host. Return the output." + child = spawn(command, + #timeout=10, + ) + if self.verbose: + sys.stderr.write("-> " + command + "\n") + seen = child.expect(self.keys) + self.f.write(str(child.before) + str(child.after)+'\n') + if seen == 0: + child.sendline('yes') + seen = child.expect(self.keys) + if seen == 1: + if not self.password: + self.password = getpass.getpass('Remote password: ') + child.sendline(self.password) + child.readline() + time.sleep(5) + # Added to allow the background running of remote process + if not child.isalive(): + seen = child.expect(self.keys) + if seen == 2: + lines = child.readlines() + self.f.write(lines) + if self.verbose: + sys.stderr.write("<- " + child.before + "|\n") + try: + self.f.write(str(child.before) + str(child.after)+'\n') + except: + pass + self.f.close() + return child.before + + def ssh(self, command): + + return self.__exec("ssh -l %s %s \"%s\"" \ + % (self.user,self.host,command)) + + def scp(self, src, dst): + + return self.__exec("scp %s %s@%s:%s" \ + % (src, session.user, session.host, dst)) + + def exists(self, file): + + "Retrieve file permissions of specified remote file." + seen = self.ssh("/bin/ls -ld %s" % file) + if string.find(seen, "No such file") > -1: + return None # File doesn't exist + else: + return seen.split()[0] # Return permission field of listing. + diff --git a/third_party/Python/module/pexpect-2.4/examples/ssh_tunnel.py b/third_party/Python/module/pexpect-2.4/examples/ssh_tunnel.py new file mode 100644 index 000000000000..3c8bc09514b4 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/ssh_tunnel.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +"""This starts an SSH tunnel to a given host. If the SSH process ever dies then +this script will detect that and restart it. I use this under Cygwin to keep +open encrypted tunnels to port 25 (SMTP), port 143 (IMAP4), and port 110 +(POP3). I set my mail client to talk to localhost and I keep this script +running in the background. + +Note that this is a rather stupid script at the moment because it just looks to +see if any ssh process is running. It should really make sure that our specific +ssh process is running. The problem is that ssh is missing a very useful +feature. It has no way to report the process id of the background daemon that +it creates with the -f command. This would be a really useful script if I could +figure a way around this problem. """ + +import pexpect +import getpass +import time + +# SMTP:25 IMAP4:143 POP3:110 +tunnel_command = 'ssh -C -N -f -L 25:127.0.0.1:25 -L 143:127.0.0.1:143 -L 110:127.0.0.1:110 %(user)@%(host)' +host = raw_input('Hostname: ') +user = raw_input('Username: ') +X = getpass.getpass('Password: ') + +def get_process_info (): + + # This seems to work on both Linux and BSD, but should otherwise be considered highly UNportable. + + ps = pexpect.run ('ps ax -O ppid') + pass +def start_tunnel (): + try: + ssh_tunnel = pexpect.spawn (tunnel_command % globals()) + ssh_tunnel.expect ('password:') + time.sleep (0.1) + ssh_tunnel.sendline (X) + time.sleep (60) # Cygwin is slow to update process status. + ssh_tunnel.expect (pexpect.EOF) + + except Exception, e: + print str(e) + +def main (): + + while True: + ps = pexpect.spawn ('ps') + time.sleep (1) + index = ps.expect (['/usr/bin/ssh', pexpect.EOF, pexpect.TIMEOUT]) + if index == 2: + print 'TIMEOUT in ps command...' + print str(ps) + time.sleep (13) + if index == 1: + print time.asctime(), + print 'restarting tunnel' + start_tunnel () + time.sleep (11) + print 'tunnel OK' + else: + # print 'tunnel OK' + time.sleep (7) + +if __name__ == '__main__': + main () + +# This was for older SSH versions that didn't have -f option +#tunnel_command = 'ssh -C -n -L 25:%(host)s:25 -L 110:%(host)s:110 %(user)s@%(host)s -f nothing.sh' +#nothing_script = """#!/bin/sh +#while true; do sleep 53; done +#""" + diff --git a/third_party/Python/module/pexpect-2.4/examples/sshls.py b/third_party/Python/module/pexpect-2.4/examples/sshls.py new file mode 100644 index 000000000000..ef1ab9c23cd5 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/sshls.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +"""This runs 'ls -l' on a remote host using SSH. At the prompts enter hostname, +user, and password. + +$Id: sshls.py 489 2007-11-28 23:40:34Z noah $ +""" + +import pexpect +import getpass, os + +def ssh_command (user, host, password, command): + + """This runs a command on the remote host. This could also be done with the +pxssh class, but this demonstrates what that class does at a simpler level. +This returns a pexpect.spawn object. This handles the case when you try to +connect to a new host and ssh asks you if you want to accept the public key +fingerprint and continue connecting. """ + + ssh_newkey = 'Are you sure you want to continue connecting' + child = pexpect.spawn('ssh -l %s %s %s'%(user, host, command)) + i = child.expect([pexpect.TIMEOUT, ssh_newkey, 'password: ']) + if i == 0: # Timeout + print 'ERROR!' + print 'SSH could not login. Here is what SSH said:' + print child.before, child.after + return None + if i == 1: # SSH does not have the public key. Just accept it. + child.sendline ('yes') + child.expect ('password: ') + i = child.expect([pexpect.TIMEOUT, 'password: ']) + if i == 0: # Timeout + print 'ERROR!' + print 'SSH could not login. Here is what SSH said:' + print child.before, child.after + return None + child.sendline(password) + return child + +def main (): + + host = raw_input('Hostname: ') + user = raw_input('User: ') + password = getpass.getpass('Password: ') + child = ssh_command (user, host, password, '/bin/ls -l') + child.expect(pexpect.EOF) + print child.before + +if __name__ == '__main__': + try: + main() + except Exception, e: + print str(e) + traceback.print_exc() + os._exit(1) + diff --git a/third_party/Python/module/pexpect-2.4/examples/table_test.html b/third_party/Python/module/pexpect-2.4/examples/table_test.html new file mode 100644 index 000000000000..5dba0ecf0c89 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/table_test.html @@ -0,0 +1,106 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"> +<html> +<head> +<title>TEST</title> +</head> +<style type="text/css"> +a {color: #9f9; text-decoration: none} +a:hover {color: #0f0} +hr {color: #0f0} +html,table,body,textarea,input,form +{ +font-family: "Courier New", Courier, mono; +font-size: 8pt; +color: #0c0; +background-color: #020; +margin:0; +padding:0; +border:0; +} +input { background-color: #010; } +textarea { +border-width:1; +border-style:solid; +border-color:#0c0; +padding:3; +margin:3; +} +</style> +<script> +var foo="" + +" 123456789012345678901234567890123456789012345 789012345678901234567890123456789"+ +"0 2345678901234567890123456789012345678901234 6 89012345678901234567890123456789"+ +"01 34567890123456789012345678901234567890123 567 9012345678901234567890123456789"+ +"012 456789012345678901234567890123456789012 45678 012345678901234567890123456789"+ +"0123 5678901234567890123456789012345678901 3456789 12345678901234567890123456789"+ +"01234 67890123456789012345678901234567890 234567890 2345678901234567890123456789"+ +"012345 789012345678901234567890123456789 12345678901 345678901234567890123456789"+ +"0123456 8901234567890123456789012345678 0123456789012 45678901234567890123456789"+ +"01234567 90123456789012345678901234567 901234567890123 5678901234567890123456789"+ +"012345678 012345678901234567890123456 89012345678901234 678901234567890123456789"+ +"0123456789 1234567890123456789012345 7890123456789012345 78901234567890123456789"+ +"01234567890 23456789012345678901234 678901234567890123456 8901234567890123456789"+ +"012345678901 345678901234567890123 56789012345678901234567 901234567890123456789"+ +"0123456789012 4567890123456789012 4567890123456789012345678 0123456789012345678 "+ +"01234567890123 56789012345678901 345678901234567890123456789 12345678901234567 9"+ +"012345678901234 678901234567890 23456789012 567 01234567890 234567890123456 89"+ +"0123456789012345 7890123456789 123457789012 567 012345678901 3456789012345 789"+ +"01234567890123456 89012345678 012345678901234567890123456789012 45678901234 6789"+ +"012345678901234567 901234567 90123456789 12345678901 34567890123 567890123 56789"+ +"0123456789012345678 0123456 8901234567890 3456789 2345678901234 6789012 456789"+ +"01234567890123456789 12345 7890123456789012 0123456789012345 78901 3456789"+ +"012345678901234567890 234 67890123456789012345678901234567890123456 890 23456789"+ +"0123456789012345678901 3 5678901234567890123456789012345678901234567 9 123456789"+ +"01234567890123456789012 456789012345678901234567890123456789012345678 0123456789"; +function start2() +{ + // get the reference for the body + //var mybody = document.getElementsByTagName("body")[0]; + var mybody = document.getElementById("replace_me"); + var myroot = document.getElementById("a_parent"); + mytable = document.createElement("table"); + mytablebody = document.createElement("tbody"); + mytable.setAttribute("border","0"); + mytable.setAttribute("cellspacing","0"); + mytable.setAttribute("cellpadding","0"); + for(var j = 0; j < 24; j++) + { + mycurrent_row = document.createElement("tr"); + for(var i = 0; i < 80; i++) + { + mycurrent_cell = document.createElement("td"); + offset = (j*80)+i; + currenttext = document.createTextNode(foo.substring(offset,offset+1)); + mycurrent_cell.appendChild(currenttext); + mycurrent_row.appendChild(mycurrent_cell); + } + mytablebody.appendChild(mycurrent_row); + } + mytable.appendChild(mytablebody); + myroot.replaceChild(mytable,mybody); + //mybody.appendChild(mytable); +} +</script> +<body onload="start2();"> +<table align="LEFT" border="0" cellspacing="0" cellpadding="0"> +<div id="a_parent"> +<span id="replace_me"> +<tr align="left" valign="left"> + <td>/</td> + <td>h</td> + <td>o</td> + <td>m</td> + <td>e</td> + <td>/</td> + <td>n</td> + <td>o</td> + <td>a</td> + <td>h</td> + <td>/</td> + <td> </td> +</tr> +</table> +</span> +</div> +</body> +</html>
\ No newline at end of file diff --git a/third_party/Python/module/pexpect-2.4/examples/topip.py b/third_party/Python/module/pexpect-2.4/examples/topip.py new file mode 100644 index 000000000000..5bd63e2ef224 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/topip.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python + +""" This runs netstat on a local or remote server. It calculates some simple +statistical information on the number of external inet connections. It groups +by IP address. This can be used to detect if one IP address is taking up an +excessive number of connections. It can also send an email alert if a given IP +address exceeds a threshold between runs of the script. This script can be used +as a drop-in Munin plugin or it can be used stand-alone from cron. I used this +on a busy web server that would sometimes get hit with denial of service +attacks. This made it easy to see if a script was opening many multiple +connections. A typical browser would open fewer than 10 connections at once. A +script might open over 100 simultaneous connections. + +./topip.py [-s server_hostname] [-u username] [-p password] {-a from_addr,to_addr} {-n N} {-v} {--ipv6} + + -s : hostname of the remote server to login to. + -u : username to user for login. + -p : password to user for login. + -n : print stddev for the the number of the top 'N' ipaddresses. + -v : verbose - print stats and list of top ipaddresses. + -a : send alert if stddev goes over 20. + -l : to log message to /var/log/topip.log + --ipv6 : this parses netstat output that includes ipv6 format. + Note that this actually only works with ipv4 addresses, but for versions of + netstat that print in ipv6 format. + --stdev=N : Where N is an integer. This sets the trigger point for alerts and logs. + Default is to trigger if max value is above 5 standard deviations. + +Example: + + This will print stats for the top IP addresses connected to the given host: + + ./topip.py -s www.example.com -u mylogin -p mypassword -n 10 -v + + This will send an alert email if the maxip goes over the stddev trigger value and + the the current top ip is the same as the last top ip (/tmp/topip.last): + + ./topip.py -s www.example.com -u mylogin -p mypassword -n 10 -v -a alert@example.com,user@example.com + + This will print the connection stats for the localhost in Munin format: + + ./topip.py + +Noah Spurrier + +$Id: topip.py 489 2007-11-28 23:40:34Z noah $ +""" + +import pexpect, pxssh # See http://pexpect.sourceforge.net/ +import os, sys, time, re, getopt, pickle, getpass, smtplib +import traceback +from pprint import pprint + +TOPIP_LOG_FILE = '/var/log/topip.log' +TOPIP_LAST_RUN_STATS = '/var/run/topip.last' + +def exit_with_usage(): + + print globals()['__doc__'] + os._exit(1) + +def stats(r): + + """This returns a dict of the median, average, standard deviation, min and max of the given sequence. + + >>> from topip import stats + >>> print stats([5,6,8,9]) + {'med': 8, 'max': 9, 'avg': 7.0, 'stddev': 1.5811388300841898, 'min': 5} + >>> print stats([1000,1006,1008,1014]) + {'med': 1008, 'max': 1014, 'avg': 1007.0, 'stddev': 5.0, 'min': 1000} + >>> print stats([1,3,4,5,18,16,4,3,3,5,13]) + {'med': 4, 'max': 18, 'avg': 6.8181818181818183, 'stddev': 5.6216817577237475, 'min': 1} + >>> print stats([1,3,4,5,18,16,4,3,3,5,13,14,5,6,7,8,7,6,6,7,5,6,4,14,7]) + {'med': 6, 'max': 18, 'avg': 7.0800000000000001, 'stddev': 4.3259218670706474, 'min': 1} + """ + + total = sum(r) + avg = float(total)/float(len(r)) + sdsq = sum([(i-avg)**2 for i in r]) + s = list(r) + s.sort() + return dict(zip(['med', 'avg', 'stddev', 'min', 'max'] , (s[len(s)//2], avg, (sdsq/len(r))**.5, min(r), max(r)))) + +def send_alert (message, subject, addr_from, addr_to, smtp_server='localhost'): + + """This sends an email alert. + """ + + message = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n' % (addr_from, addr_to, subject) + message + server = smtplib.SMTP(smtp_server) + server.sendmail(addr_from, addr_to, message) + server.quit() + +def main(): + + ###################################################################### + ## Parse the options, arguments, etc. + ###################################################################### + try: + optlist, args = getopt.getopt(sys.argv[1:], 'h?valqs:u:p:n:', ['help','h','?','ipv6','stddev=']) + except Exception, e: + print str(e) + exit_with_usage() + options = dict(optlist) + + munin_flag = False + if len(args) > 0: + if args[0] == 'config': + print 'graph_title Netstat Connections per IP' + print 'graph_vlabel Socket connections per IP' + print 'connections_max.label max' + print 'connections_max.info Maximum number of connections per IP' + print 'connections_avg.label avg' + print 'connections_avg.info Average number of connections per IP' + print 'connections_stddev.label stddev' + print 'connections_stddev.info Standard deviation' + return 0 + elif args[0] != '': + print args, len(args) + return 0 + exit_with_usage() + if [elem for elem in options if elem in ['-h','--h','-?','--?','--help']]: + print 'Help:' + exit_with_usage() + if '-s' in options: + hostname = options['-s'] + else: + # if host was not specified then assume localhost munin plugin. + munin_flag = True + hostname = 'localhost' + # If localhost then don't ask for username/password. + if hostname != 'localhost' and hostname != '127.0.0.1': + if '-u' in options: + username = options['-u'] + else: + username = raw_input('username: ') + if '-p' in options: + password = options['-p'] + else: + password = getpass.getpass('password: ') + else: + use_localhost = True + + if '-l' in options: + log_flag = True + else: + log_flag = False + if '-n' in options: + average_n = int(options['-n']) + else: + average_n = None + if '-v' in options: + verbose = True + else: + verbose = False + if '-a' in options: + alert_flag = True + (alert_addr_from, alert_addr_to) = tuple(options['-a'].split(',')) + else: + alert_flag = False + if '--ipv6' in options: + ipv6_flag = True + else: + ipv6_flag = False + if '--stddev' in options: + stddev_trigger = float(options['--stddev']) + else: + stddev_trigger = 5 + + if ipv6_flag: + netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+::ffff:(\S+):(\S+)\s+.*?\r' + else: + netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(?:::ffff:)*(\S+):(\S+)\s+.*?\r' + #netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+):(\S+)\s+.*?\r' + + # run netstat (either locally or via SSH). + if use_localhost: + p = pexpect.spawn('netstat -n -t') + PROMPT = pexpect.TIMEOUT + else: + p = pxssh.pxssh() + p.login(hostname, username, password) + p.sendline('netstat -n -t') + PROMPT = p.PROMPT + + # loop through each matching netstat_pattern and put the ip address in the list. + ip_list = {} + try: + while 1: + i = p.expect([PROMPT, netstat_pattern]) + if i == 0: + break + k = p.match.groups()[4] + if k in ip_list: + ip_list[k] = ip_list[k] + 1 + else: + ip_list[k] = 1 + except: + pass + + # remove a few common, uninteresting addresses from the dictionary. + ip_list = dict([ (key,value) for key,value in ip_list.items() if '192.168.' not in key]) + ip_list = dict([ (key,value) for key,value in ip_list.items() if '127.0.0.1' not in key]) + + # sort dict by value (count) + #ip_list = sorted(ip_list.iteritems(),lambda x,y:cmp(x[1], y[1]),reverse=True) + ip_list = ip_list.items() + if len(ip_list) < 1: + if verbose: print 'Warning: no networks connections worth looking at.' + return 0 + ip_list.sort(lambda x,y:cmp(y[1],x[1])) + + # generate some stats for the ip addresses found. + if average_n <= 1: + average_n = None + s = stats(zip(*ip_list[0:average_n])[1]) # The * unary operator treats the list elements as arguments + s['maxip'] = ip_list[0] + + # print munin-style or verbose results for the stats. + if munin_flag: + print 'connections_max.value', s['max'] + print 'connections_avg.value', s['avg'] + print 'connections_stddev.value', s['stddev'] + return 0 + if verbose: + pprint (s) + print + pprint (ip_list[0:average_n]) + + # load the stats from the last run. + try: + last_stats = pickle.load(file(TOPIP_LAST_RUN_STATS)) + except: + last_stats = {'maxip':None} + + if s['maxip'][1] > (s['stddev'] * stddev_trigger) and s['maxip']==last_stats['maxip']: + if verbose: print 'The maxip has been above trigger for two consecutive samples.' + if alert_flag: + if verbose: print 'SENDING ALERT EMAIL' + send_alert(str(s), 'ALERT on %s' % hostname, alert_addr_from, alert_addr_to) + if log_flag: + if verbose: print 'LOGGING THIS EVENT' + fout = file(TOPIP_LOG_FILE,'a') + #dts = time.strftime('%Y:%m:%d:%H:%M:%S', time.localtime()) + dts = time.asctime() + fout.write ('%s - %d connections from %s\n' % (dts,s['maxip'][1],str(s['maxip'][0]))) + fout.close() + + # save state to TOPIP_LAST_RUN_STATS + try: + pickle.dump(s, file(TOPIP_LAST_RUN_STATS,'w')) + os.chmod (TOPIP_LAST_RUN_STATS, 0664) + except: + pass + # p.logout() + +if __name__ == '__main__': + try: + main() + sys.exit(0) + except SystemExit, e: + raise e + except Exception, e: + print str(e) + traceback.print_exc() + os._exit(1) + diff --git a/third_party/Python/module/pexpect-2.4/examples/uptime.py b/third_party/Python/module/pexpect-2.4/examples/uptime.py new file mode 100644 index 000000000000..f5018dfe0c18 --- /dev/null +++ b/third_party/Python/module/pexpect-2.4/examples/uptime.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +"""This displays uptime information using uptime. This is redundant, +but it demonstrates expecting for a regular expression that uses subgroups. + +$Id: uptime.py 489 2007-11-28 23:40:34Z noah $ +""" + +import pexpect +import re + +# There are many different styles of uptime results. I try to parse them all. Yeee! +# Examples from different machines: +# [x86] Linux 2.4 (Redhat 7.3) +# 2:06pm up 63 days, 18 min, 3 users, load average: 0.32, 0.08, 0.02 +# [x86] Linux 2.4.18-14 (Redhat 8.0) +# 3:07pm up 29 min, 1 user, load average: 2.44, 2.51, 1.57 +# [PPC - G4] MacOS X 10.1 SERVER Edition +# 2:11PM up 3 days, 13:50, 3 users, load averages: 0.01, 0.00, 0.00 +# [powerpc] Darwin v1-58.corefa.com 8.2.0 Darwin Kernel Version 8.2.0 +# 10:35 up 18:06, 4 users, load averages: 0.52 0.47 0.36 +# [Sparc - R220] Sun Solaris (8) +# 2:13pm up 22 min(s), 1 user, load average: 0.02, 0.01, 0.01 +# [x86] Linux 2.4.18-14 (Redhat 8) +# 11:36pm up 4 days, 17:58, 1 user, load average: 0.03, 0.01, 0.00 +# AIX jwdir 2 5 0001DBFA4C00 +# 09:43AM up 23:27, 1 user, load average: 0.49, 0.32, 0.23 +# OpenBSD box3 2.9 GENERIC#653 i386 +# 6:08PM up 4 days, 22:26, 1 user, load averages: 0.13, 0.09, 0.08 + +# This parses uptime output into the major groups using regex group matching. +p = pexpect.spawn ('uptime') +p.expect('up\s+(.*?),\s+([0-9]+) users?,\s+load averages?: ([0-9]+\.[0-9][0-9]),?\s+([0-9]+\.[0-9][0-9]),?\s+([0-9]+\.[0-9][0-9])') +duration, users, av1, av5, av15 = p.match.groups() + +# The duration is a little harder to parse because of all the different +# styles of uptime. I'm sure there is a way to do this all at once with +# one single regex, but I bet it would be hard to read and maintain. +# If anyone wants to send me a version using a single regex I'd be happy to see it. +days = '0' +hours = '0' +mins = '0' +if 'day' in duration: + p.match = re.search('([0-9]+)\s+day',duration) + days = str(int(p.match.group(1))) +if ':' in duration: + p.match = re.search('([0-9]+):([0-9]+)',duration) + hours = str(int(p.match.group(1))) + mins = str(int(p.match.group(2))) +if 'min' in duration: + p.match = re.search('([0-9]+)\s+min',duration) + mins = str(int(p.match.group(1))) + +# Print the parsed fields in CSV format. +print 'days, hours, minutes, users, cpu avg 1 min, cpu avg 5 min, cpu avg 15 min' +print '%s, %s, %s, %s, %s, %s, %s' % (days, hours, mins, users, av1, av5, av15) + |