KaneAI - World’s First E2E Software Testing Agent.Book a Demo 
How to use print_to_console method in grail
Best Python code snippet using grail_python
setup_OpenELIS.py
Source:setup_OpenELIS.py 
1#!/usr/bin/env python2# -*- coding: utf-8 -*-3import glob4import os5import sys6import string7import shutil8import stat9import time10import re11from time import gmtime, strftime12from random import Random13VERSION = "7.0"14TEMPLATE_DIR = "./templates/"15WAR_DIR = "./warFiles/"16STAGING_DIR = "./stagingFiles/"17LIQUIBASE_DIR = "./liquibase/"18DB_DIR = "./databaseFiles/"19CROSSTAB_DIR = "./crosstab/"20ROLLBACK_DIR = "./rollback/"21LOG_DIR = "./log/"22FIX_DIR = "./fixes/"23APT_CACHE_DIR = "/var/cache/apt/archives/"24EXPECTED_CROSSTAB_FUNCTIONS = "3"25BIN_DIR = "./bin/"26CRON_FILE_DIR = "./cronFiles/"27CRON_INSTALL_DIR = "/etc/cron.d/"28CRON_FILE = "openElis"29POSTGRES_LIB_DIR = "/usr/lib/postgresql/8.3/lib/"30BACKUP_DIR = os.getenv("HOME") + "/openElisBackup/"31BACKUP_SCRIPT_NAME = "DatabaseBackup.pl"32LOG_FILE_NAME = "installer.log"33LANG_NAME = "en_US.UTF-8"34POSTGRES_ROLE_UPDATE_FILE_NAME = "updateDBPassword.sql"35APP_NAME = ""36REPORT_IMAGE_PATH = "/WEB-INF/reports/images/"37PREVIEW_IMAGE_PATH = "/images/"38PLUGIN_PATH = "/plugin"39# maps the application name to the liquibase context name40APP_CONTEX_MAP = {'CDI': 'CDIRetroCI', 'LNSP_Haiti': 'haitiLNSP', 'haiti': 'haiti', 'CI_LNSP': 'ciLNSP',41 'CI_IPCI': 'CI_IPCI', 'CDI_RegLab': 'ci_regional', 'Kenya': 'Kenya'}42CLINLIMS_PWD = ''43ADMIN_PWD = ''44SITE_ID = ''45LOG_FILE = ''46PRINT_TO_CONSOLE = True47ACTION_TIME = ''48TOMCAT_DIR = ''49TOMCAT_BASE = '/usr/share/tomcat'50TOMCAT_VERSION = ''51TOMCAT_INSTALL_DIR = '/usr/share/'52NO_PROMPT = False53# note There is a fair amount of copying files, it should be re-written using shutil54def install_crosstab():55 log("Checking if crosstab is installed in Postgres", True)56 # this creates an empty file to write to57 result = open(STAGING_DIR + 'crosstabResult.txt', 'wr')58 result.close()59 os.chmod(STAGING_DIR + 'crosstabResult.txt', stat.S_IROTH | stat.S_IWOTH)60 cmd = cmd = 'su -c "psql clinlims -L ' + STAGING_DIR + 'crosstabResult.txt < ' + CROSSTAB_DIR + 'crosstabCheck.sql" postgres > /dev/null'61 os.system(cmd)62 check_file = open(STAGING_DIR + 'crosstabResult.txt')63 for line in check_file:64 if line.find(EXPECTED_CROSSTAB_FUNCTIONS) != -1:65 log("Crosstabs are installed", True)66 return67 log("Installing crosstabs", True)68 # if tablefunc.so is not present in POSTGRES_LIB_DIR then copy it from CROSSTAB_DIR69 # the version in CROSSTAB_DIR is for 8.3 (assumes all new installs will be VM based)70 if not os.path.exists(POSTGRES_LIB_DIR + "tablefunc.so") and "8.3" in POSTGRES_LIB_DIR:71 # copy the lib file to the lib directory72 shutil.copy(CROSSTAB_DIR + "tablefunc.so", POSTGRES_LIB_DIR)73 # run the installer74 cmd = cmd = 'su -c "psql clinlims < ' + CROSSTAB_DIR + 'tablefunc.sql" postgres'75 os.system(cmd)76def check_preconditions():77 """Checks that the correct versions of tomcat, postgres and jre are installed. Note that the jre check is not very robust"""78 global TOMCAT_VERSION79 global TOMCAT_DIR80 global POSTGRES_LIB_DIR81 log("Checking for Tomcat 5.5 or later installation", PRINT_TO_CONSOLE)82 TOMCAT_DIR = get_tomcat_directory()83 TOMCAT_VERSION = TOMCAT_DIR.strip(TOMCAT_INSTALL_DIR)84 log("Tomcat found version = " + TOMCAT_VERSION, PRINT_TO_CONSOLE)85 log("Checking for Postgres 8.3 or later installation", PRINT_TO_CONSOLE)86 if not check_postgres_preconditions():87 return False88 log("Checking for correcting for correct java runtime", PRINT_TO_CONSOLE)89 # do rough check for debian what kind of configuration to do. This is the not a robust part90 jvm_six_path = "/usr/lib/jvm/java-6-sun/jre/bin/"91 if os.path.exists(jvm_six_path):92 update_alternative_path = "/usr/sbin/update-alternatives"93 if os.path.exists(update_alternative_path):94 set_java_path = update_alternative_path + " --set java " + jvm_six_path + "java"95 set_key_path = update_alternative_path + " --set keytool " + jvm_six_path + "keytool"96 os.system(set_java_path)97 os.system(set_key_path)98 if TOMCAT_VERSION == "tomcat5.5":99 update_script = "./configTomcat.sh "100 os.system("chmod +x " + update_script)101 os.system(update_script + " > /dev/null ")102 return True103def current_installation():104 name = find_installed_name()105 return os.path.exists(TOMCAT_DIR + '/conf/Catalina/localhost/' + name + '.xml')106def get_tomcat_directory():107 names = glob.glob(TOMCAT_BASE + '[0-9].[0-9]')108 if names:109 version = names[0].strip(TOMCAT_BASE)110 split_version = version.split('.')111 major = int(split_version[0])112 minor = int(split_version[1])113 # check for version 5.5 or greater114 if major == 5 and minor >= 5:115 log("Found " + names[0], PRINT_TO_CONSOLE)116 return names[0]117 elif major > 5:118 log("Found " + names[0], PRINT_TO_CONSOLE)119 return names[0]120 else:121 log("Tomcat must be version 5.5 or later\n", PRINT_TO_CONSOLE)122 return None123 names = glob.glob(TOMCAT_BASE + '[0-9]')124 if names:125 version = names[0].strip(TOMCAT_BASE)126 major = int(version)127 if major > 5:128 log("Found " + names[0], PRINT_TO_CONSOLE)129 return names[0]130 else:131 log("Tomcat must be version 5.5 or later\n", PRINT_TO_CONSOLE)132 return None133 return None134def config_files_for_postgres():135 global ADMIN_PWD136 if NO_PROMPT:137 pass138 else:139 ADMIN_PWD = ''.join(Random().sample(string.letters, 12))140 print "This is the postgres admin password. Please record it in a safe and private place."141 print "It will not be able to be recovered once this script is finished\n"142 print ADMIN_PWD143 print raw_input("\npress any key once you have recorded it")144 os.system('clear')145 # set values for database users146 pg_permissions = open(TEMPLATE_DIR + 'pgsql-permissions.sql')147 output_file = open(STAGING_DIR + 'pgsqlPermissions.sql', 'w')148 for line in pg_permissions:149 if line.find('itechappPassword') > 0:150 line = line.replace('[% itechappPassword %]', CLINLIMS_PWD)151 output_file.write(line)152 elif line.find('adminPassword') > 0:153 line = line.replace('[% adminPassword %]', ADMIN_PWD)154 output_file.write(line)155 else:156 output_file.write(line)157 output_file.close()158 pg_permissions.close()159def get_file_name(file):160 filename_parts = file.split('/')161 return filename_parts[len(filename_parts) - 1]162def find_installation_name():163 global APP_NAME164 war_files = glob.glob(WAR_DIR + '*.war')165 for file in war_files:166 filename = get_file_name(file)167 if re.match('.*OpenElis.war', filename):168 APP_NAME = filename.split('.')[0]169 return True170 return False171def find_installed_name():172 global TOMCAT_DIR173 global TOMCAT_VERSION174 TOMCAT_DIR = get_tomcat_directory()175 TOMCAT_VERSION = TOMCAT_DIR.strip(TOMCAT_INSTALL_DIR)176 war_files = glob.glob(TOMCAT_DIR + '/webapps/*.war')177 for file in war_files:178 filename = get_file_name(file)179 if re.match('.*OpenElis.war', filename):180 return filename.split('.')[0]181 return ''182def config_files_for_tomcat():183 openelis_template_file = open(TEMPLATE_DIR + 'openelis.xml')184 openelis_config_file = open(STAGING_DIR + APP_NAME + '.xml', 'w')185 for line in openelis_template_file:186 if line.find('itechappPassword') > 0:187 line = line.replace('[% itechappPassword %]', CLINLIMS_PWD)188 elif line.find('appname') > 0:189 openelis_line = line.replace('[% appname %]', APP_NAME)190 openelis_config_file.write(openelis_line)191 continue192 openelis_config_file.write(line)193 openelis_config_file.close()194 openelis_template_file.close()195def config_site_information():196 global SITE_ID197 # Get site specific information198 print """199 Some installations require configuuration. 200 You will be asked for specific information which may or may not be needed for this installation.201 If you do not know if it is needed or you do not know the correct value it may be left blank.202 You can set the values after the installation is complete.203 """204 if NO_PROMPT:205 pass206 else:207 site_file = open(STAGING_DIR + 'siteInfo.sql', 'w')208 SITE_ID = raw_input("site number for this lab (5 character): ")209 persist_site_information(site_file, 'siteNumber', 'The site number of the this lab', SITE_ID)210 persist_site_information(site_file, 'Accession number prefix',211 'Prefix for SITEYEARNUM format. Can not be changed if there are samples', SITE_ID)212 site_file.close()213def check_postgres_preconditions():214 log("Checking for Postgres 8.3 or later installation", PRINT_TO_CONSOLE)215 os.system('psql --version > tmp')216 tmp_file = open('tmp')217 first_line = tmp_file.readline()218 tmp_file.close()219 if len(first_line) < 1:220 log("Postgress not installed. Please install postgres8.3 or later", PRINT_TO_CONSOLE)221 return False222 split_line = first_line.split()223 version = split_line[2]224 split_version = version.split('.')225 valid = False226 if split_version[0].isdigit() and split_version[1].isdigit():227 major = int(split_version[0])228 minor = int(split_version[1])229 valid = major > 8 or (major == 8 and minor >= 3)230 if valid:231 log("Postgres" + str(major) + "." + str(minor) + " found!\n", PRINT_TO_CONSOLE)232 POSTGRES_LIB_DIR = "/usr/lib/postgresql/" + str(major) + "." + str(minor) + "/lib/"233 return True234 else:235 log("Postgres must be 8.3 or later\n", PRINT_TO_CONSOLE)236 return False237def persist_site_information(file, name, description, value):238 DELETE_PRE = 'DELETE from clinlims.site_information where name = \''239 DELETE_POST = '\';\n'240 INSERT_PRE = 'INSERT INTO clinlims.site_information( id, lastupdated, "name", description, "value") VALUES ( nextval(\'clinlims.site_information_seq\'), now(), \''241 INSERT_POST = ' );\n\n'242 delete_line = DELETE_PRE + name + DELETE_POST243 insert_value = 'null'244 if len(value) > 0:245 insert_value = '\'' + value + '\''246 insert_line = INSERT_PRE + name + '\', \'' + description + '\', ' + insert_value + INSERT_POST247 file.write(delete_line)248 file.write(insert_line)249def install_db():250 cmd = 'su -c "psql < ' + STAGING_DIR + 'pgsqlPermissions.sql" postgres'251 os.system(cmd)252 cmd = 'su -c "psql clinlims < ' + DB_DIR + 'databaseInstall.backup" postgres'253 os.system(cmd)254 cmd = 'su -c "psql clinlims < ' + STAGING_DIR + 'siteInfo.sql" postgres'255 os.system(cmd)256def install_backup():257 global SITE_ID258 if NO_PROMPT:259 pass260 else:261 if os.path.exists(BACKUP_DIR + BACKUP_SCRIPT_NAME):262 over_ride = raw_input("The backup script is already installed. Do you want to overwrite it? y/n ")263 if not over_ride.lower() == "y":264 return265 # make sure the template files for the backup exits266 if not os.path.exists(TEMPLATE_DIR + BACKUP_SCRIPT_NAME):267 log("Not installing backup. Script missing", PRINT_TO_CONSOLE)268 return269 else:270 log("Installing backup", True)271 # make sure curl is installed272 log("Checking for curl", True)273 if os.system('which curl') == 256:274 log('curl is not installed. http://curl.linux-mirror.org/', PRINT_TO_CONSOLE)275 return276 else:277 log("Curl found, continuing with backup installation", PRINT_TO_CONSOLE)278 if APP_NAME == 'haitiOpenElis' or not NO_PROMPT:279 if len(SITE_ID) < 1:280 SITE_ID = raw_input("site number or identification for this lab: ")281 while len(SITE_ID) < 1:282 SITE_ID = raw_input(283 "the site number is required to install the backup. If you want to stop installing the backup use a site number of 'q': ")284 if SITE_ID.lower() == 'q':285 return286 template_file = open(TEMPLATE_DIR + BACKUP_SCRIPT_NAME, "r")287 staging_file = open(STAGING_DIR + BACKUP_SCRIPT_NAME, "w")288 for line in template_file:289 if line.find("[% installName %]") > 0:290 line = line.replace("[% installName %]", APP_NAME)291 if line.find("[% siteId %]") > 0:292 line = line.replace("[% siteId %]", SITE_ID)293 if line.find("[% postgres_password %]") > 0:294 line = line.replace("[% postgres_password %]", CLINLIMS_PWD)295 staging_file.write(line)296 template_file.close()297 staging_file.close()298 if not os.path.exists(BACKUP_DIR):299 os.makedirs(BACKUP_DIR, 0755)300 if not os.path.exists(BACKUP_DIR + "daily"):301 os.makedirs(BACKUP_DIR + "daily", 0755)302 if not os.path.exists(BACKUP_DIR + "cumulative"):303 os.makedirs(BACKUP_DIR + "cumulative", 0755)304 if not os.path.exists(BACKUP_DIR + "transmissionQueue"):305 os.makedirs(BACKUP_DIR + "transmissionQueue", 0755)306 shutil.copy(STAGING_DIR + BACKUP_SCRIPT_NAME, BACKUP_DIR + BACKUP_SCRIPT_NAME)307 os.chmod(BACKUP_DIR + BACKUP_SCRIPT_NAME, 0777)308 shutil.copy(CRON_FILE_DIR + CRON_FILE, CRON_INSTALL_DIR)309 return310def update_with_liquibase():311 global LOG_FILE312 context = False313 app_names = APP_CONTEX_MAP.keys()314 for app in app_names:315 if APP_NAME.find(app) == 0:316 context = APP_CONTEX_MAP[app]317 break318 context_arg = ''319 if context:320 context_arg = "--contexts=" + context321 log("Updating database\n", True)322 LOG_FILE.flush()323 cmd = "java -jar ./lib/liquibase-1.9.5.jar --defaultsFile=liquibaseInstall.properties " + context_arg + \324 " --password=" + CLINLIMS_PWD + " --username=clinlims update >> ../" + LOG_DIR + LOG_FILE_NAME + " 2>&" + str(325 LOG_FILE.fileno())326 print "The following command may take a few minutes to complete. Please take a short break until it finishes"327 print cmd328 current_dir = os.getcwd()329 os.chdir(LIQUIBASE_DIR)330 result = os.system(cmd)331 if result > 0:332 log("Error running database update", PRINT_TO_CONSOLE)333 log("Re running with diagnostics", PRINT_TO_CONSOLE)334 log("If the issue is reported as an authentication problem run the installer with the '-recover' option", PRINT_TO_CONSOLE)335 print "For further information please check the log file " + LOG_DIR + LOG_FILE_NAME336 LOG_FILE.flush()337 cmd = "java -jar ./lib/liquibase-1.9.5.jar --defaultsFile=liquibaseInstall.properties " + context_arg + \338 " --password=" + CLINLIMS_PWD + " --username=clinlims --logLevel=fine update >> ../" + LOG_DIR + LOG_FILE_NAME + " 2>&" + str(339 LOG_FILE.fileno())340 result = os.system(cmd)341 if result > 0:342 os.chdir(current_dir)343 rollback()344 clean_exit()345 os.chdir(current_dir)346def install_no_prompt():347 global NO_PROMPT348 NO_PROMPT = True349 set_environment_variables()350 install()351def update_postgres_role():352 template_file = open(TEMPLATE_DIR + POSTGRES_ROLE_UPDATE_FILE_NAME, "r")353 staging_file = open(STAGING_DIR + POSTGRES_ROLE_UPDATE_FILE_NAME, "w")354 for line in template_file:355 if line.find("[% postgres_password %]") > 0:356 line = line.replace("[% postgres_password %]", CLINLIMS_PWD)357 staging_file.write(line)358 template_file.close()359 staging_file.close()360 cmd = 'su -c "psql < ' + STAGING_DIR + POSTGRES_ROLE_UPDATE_FILE_NAME + '" postgres > /dev/null'361 os.system(cmd)362def recover_database():363 over_ride = raw_input("This will reset the database password and may install a new war file. Do you want to keep going? y/n ")364 if not over_ride.lower() == "y":365 return366 find_installed_name()367 find_installation_name()368 stop_tomcat()369 filename = APP_NAME + '.war'370 if os.path.exists(TOMCAT_DIR + '/webapps/' + filename):371 log("Using existing war file " + filename, PRINT_TO_CONSOLE)372 else:373 copy_war_file_to_tomcat()374 log("Installing " + filename + " from installer", PRINT_TO_CONSOLE)375 create_postgres_password()376 config_files_for_tomcat()377 copy_tomcat_config_file()378 update_postgres_role()379 log("Updated postgres password in database and config file.", PRINT_TO_CONSOLE)380 start_tomcat()381def set_environment_variables():382 os.environ["LC_ALL"] = ""383 os.environ["LC_CTYPE"] = ""384def install():385 set_environment_variables()386 name = find_installed_name()387 if len(name) > 0:388 log("\nThere is a currently installed version of OpenElis: " + name, PRINT_TO_CONSOLE)389 print "Please either do an uninstall or update first"390 write_help()391 clean_exit()392 if not check_preconditions():393 clean_exit()394 do_install()395def stop_tomcat():396 try_count = 1397 if TOMCAT_VERSION != 'tomcat5.5':398 log('/etc/init.d/' + TOMCAT_VERSION + ' stop', PRINT_TO_CONSOLE)399 cmd = '/etc/init.d/' + TOMCAT_VERSION + ' stop'400 os.system(cmd)401 else:402 cmd = 'invoke-rc.d ' + TOMCAT_VERSION + ' stop &> /dev/null'403 log('Stopping Tomcat servlet engine', PRINT_TO_CONSOLE)404 while os.system(cmd) != 0:405 if try_count > 4:406 log("Unable to stop tomcat. Abandoning operation", PRINT_TO_CONSOLE)407 clean_exit()408 try_count += 1409 log('Stopping Tomcat servlet engine failed, trying again', PRINT_TO_CONSOLE)410 time.sleep(3)411def start_tomcat():412 cmd = '/etc/init.d/' + TOMCAT_VERSION + ' start'413 os.system(cmd)414def restart_tomcat():415 cmd = '/etc/init.d/' + TOMCAT_VERSION + ' restart'416 os.system(cmd)417def copy_war_file_to_tomcat():418 cmd = 'cp ' + WAR_DIR + APP_NAME + '.war ' + TOMCAT_DIR + '/webapps/'419 log(cmd, PRINT_TO_CONSOLE)420 os.system(cmd)421def backup_war_file():422 log("Backing up application to " + ROLLBACK_DIR + get_action_time() + '/' + APP_NAME + '.war ', PRINT_TO_CONSOLE)423 cmd = 'cp ' + TOMCAT_DIR + '/webapps/' + APP_NAME + '.war ' + ROLLBACK_DIR + get_action_time() + '/' + APP_NAME + '.war'424 os.system(cmd)425def backup_plugins():426 if os.path.isdir(TOMCAT_DIR + '/webapps/' + APP_NAME + PLUGIN_PATH):427 log("Backing up plugins to " + ROLLBACK_DIR + get_action_time() + PLUGIN_PATH, PRINT_TO_CONSOLE)428 shutil.copytree(TOMCAT_DIR + '/webapps/' + APP_NAME + PLUGIN_PATH, ROLLBACK_DIR + get_action_time() + '/plugin')429 else:430 log("This version of openELIS does not support plugins, can not backup", PRINT_TO_CONSOLE)431def restore_plugins():432 if os.path.isdir(ROLLBACK_DIR + get_action_time() + '/plugin/') and len(433 os.listdir(ROLLBACK_DIR + get_action_time() + '/plugin/')) > 1:434 log("Restoring " + ROLLBACK_DIR + get_action_time() + PLUGIN_PATH, PRINT_TO_CONSOLE)435 cmd = 'cp -r ' + ROLLBACK_DIR + get_action_time() + '/plugin/* ' + TOMCAT_DIR + '/webapps/' + APP_NAME + PLUGIN_PATH436 os.system(cmd)437 log("Restarting tomcat to install plugins into OpenElis", PRINT_TO_CONSOLE)438 return True439 else:440 log("No plugins to restore", PRINT_TO_CONSOLE)441 return False442def delete_war_file_from_tomcat(name):443 war_path = TOMCAT_DIR + '/webapps/' + name + '.war'444 os.remove(war_path)445def delete_app_directory_from_tomcat(name):446 cmd = 'rm -r ' + TOMCAT_DIR + '/webapps/' + name + '/'447 os.system(cmd)448def delete_work_directory_from_tomcat(name):449 cmd = 'rm -r ' + TOMCAT_DIR + '/work/Catalina/localhost/' + name + '/'450 os.system(cmd)451def delete_backup():452 if os.path.exists(BACKUP_DIR):453 log("removing backup", PRINT_TO_CONSOLE)454 shutil.rmtree(BACKUP_DIR)455 if os.path.exists(CRON_INSTALL_DIR + CRON_FILE):456 log("removing crontab", PRINT_TO_CONSOLE)457 os.remove(CRON_INSTALL_DIR + CRON_FILE)458def delete_backup_script():459 if os.path.exists(BACKUP_DIR + BACKUP_SCRIPT_NAME):460 os.remove(BACKUP_DIR + BACKUP_SCRIPT_NAME)461def do_install():462 find_installation_name()463 log("installing " + APP_NAME, PRINT_TO_CONSOLE)464 create_postgres_password()465 config_files_for_postgres()466 # prepare any site-specific information467 print APP_NAME468 if APP_NAME == 'haitiOpenElis' or APP_NAME == 'CDI_RegLabOpenElis':469 config_site_information()470 # add database users and install db471 install_db()472 # set values for tomcat473 config_files_for_tomcat()474 # run liquibase for any last minute updates475 update_with_liquibase()476 stop_tomcat()477 # copy postgres jar file478 postgres_file = glob.glob(BIN_DIR + 'postgres*')479 if TOMCAT_VERSION == "tomcat5.5":480 cmd = 'cp ' + postgres_file[0] + ' ' + TOMCAT_DIR + '/common/lib/'481 else:482 cmd = 'cp ' + postgres_file[0] + ' ' + TOMCAT_DIR + '/lib/'483 os.system(cmd)484 copy_tomcat_config_file()485 copy_war_file_to_tomcat()486 start_tomcat()487 install_backup()488 # remove the files with passwords489 staging_files = glob.glob(STAGING_DIR + '*')490 for file in staging_files:491 cmd = "rm " + file492 os.system(cmd)493def create_postgres_password():494 global CLINLIMS_PWD495 CLINLIMS_PWD = ''.join(Random().sample(string.letters, 12))496def copy_tomcat_config_file():497 cmd = 'cp ' + STAGING_DIR + APP_NAME + '.xml ' + TOMCAT_DIR + '/conf/Catalina/localhost/'498 os.system(cmd)499 if TOMCAT_VERSION == "tomcat5.5":500 cmd = 'chown tomcat55:nogroup ' + TOMCAT_DIR + '/conf/Catalina/localhost/' + APP_NAME + '.xml'501 else:502 cmd = 'chown ' + TOMCAT_VERSION + ":" + TOMCAT_VERSION + ' ' + TOMCAT_DIR + '/conf/Catalina/localhost/' + APP_NAME + '.xml'503 os.system(cmd)504def do_uninstall():505 global APP_NAME506 log("checking for current installation", PRINT_TO_CONSOLE)507 installed_name = find_installed_name()508 if len(installed_name) == 0:509 log("Unable to find installed OpenELIS in tomcat", PRINT_TO_CONSOLE)510 print "The database can **NOT** be backed up\n"511 remove_db = raw_input("Do you want to remove the database? y/n")512 if remove_db.lower() == 'y':513 log("User selected to remove database", not PRINT_TO_CONSOLE)514 delete_database()515 return516 if not find_installation_name():517 log('Unable to find name of application in installer, exiting', PRINT_TO_CONSOLE)518 return519 if not APP_NAME == installed_name:520 log('Installed name: ' + installed_name + " does not match name in installer: " + APP_NAME, PRINT_TO_CONSOLE)521 remove = raw_input("Do you want to remove " + installed_name + "? y/n")522 if not (remove.lower() == 'y'):523 return524 else:525 remove = raw_input("Do you want to remove " + APP_NAME + " from tomcat? y/n ")526 if not remove.lower() == 'y':527 return528 APP_NAME = installed_name529 stop_tomcat()530 backup_db()531 backup_war_file()532 backup_config_file()533 log("removing " + APP_NAME, PRINT_TO_CONSOLE)534 config_file = TOMCAT_DIR + '/conf/Catalina/localhost/' + APP_NAME + '.xml'535 if not os.path.isfile(config_file):536 log("unable to find config file, continuing", not PRINT_TO_CONSOLE)537 else:538 os.remove(config_file)539 delete_war_file_from_tomcat(APP_NAME)540 delete_app_directory_from_tomcat(APP_NAME)541 do_delete_backup = raw_input("Do you want to remove backupfiles from this machines y/n ")542 if do_delete_backup.lower() == 'y':543 delete_backup()544 delete_backup_script()545 delete_database()546 start_tomcat()547def delete_database():548 log("Dropping clinlims database and OpenELIS database roles", PRINT_TO_CONSOLE)549 os.system('su -c "dropdb -e clinlims" postgres')550 os.system('su -c "dropuser -e clinlims" postgres')551 os.system('su -c "dropuser -e admin" postgres')552def check_update_preconditions():553 if not check_postgres_preconditions():554 return False555 global TOMCAT_DIR556 global TOMCAT_VERSION557 TOMCAT_DIR = get_tomcat_directory()558 TOMCAT_VERSION = TOMCAT_DIR.strip(TOMCAT_INSTALL_DIR)559 # get name of existing war files560 webapps = glob.glob(TOMCAT_DIR + '/webapps/*.war')561 # get name of new war file562 find_installation_name()563 matching_war = APP_NAME + ".war"564 # return their compare value565 for file in webapps:566 if matching_war == get_file_name(file):567 return True568 return False569def find_password(app_name):570 global CLINLIMS_PWD571 config_file = open(TOMCAT_DIR + '/conf/Catalina/localhost/' + app_name + '.xml')572 for line in config_file:573 password_index = line.find("password")574 if password_index > 0:575 password_values = line[password_index:].split("\"")576 CLINLIMS_PWD = password_values[1]577 return True578def apply_separate_test_fix():579 if not os.path.exists(FIX_DIR + "collapseTests/packages/python-pgsql_2.5.1-2+b2_i386.deb"):580 log("Note: Not applying separateTestFix.", PRINT_TO_CONSOLE)581 return582 matches = glob.glob(os.path.join(APT_CACHE_DIR, "python-pgsql*"))583 # The only systems this is being applied to are those where the module has not584 # yet been installed so if it is installed it is assumed that the fix has been run already585 if len(matches) > 0:586 log("Separate Test Fix already run", PRINT_TO_CONSOLE)587 return588 log("Apply Separate Test Fix", PRINT_TO_CONSOLE)589 packages = glob.glob(FIX_DIR + "collapseTests/packages/*")590 for package in packages:591 shutil.copy(package, APT_CACHE_DIR)592 cmd = "dpkg -i " + FIX_DIR + "collapseTests/packages/*"593 os.system(cmd)594 cmd = "python " + FIX_DIR + "collapseTests/testReconfiguration_Linux.py -p " + CLINLIMS_PWD + " -d clinlims"595 os.system(cmd)596 log("Fix applied", PRINT_TO_CONSOLE)597def get_action_time():598 global ACTION_TIME599 if ACTION_TIME == '':600 ACTION_TIME = strftime("%Y_%m_%d-%H_%M_%S", time.localtime())601 cmd = 'mkdir ' + ROLLBACK_DIR + '/' + ACTION_TIME602 os.system(cmd)603 return ACTION_TIME604def backup_db():605 backup_name = get_action_time() + '/openElis.backup'606 find_password(find_installed_name())607 log("backing up database to " + ROLLBACK_DIR + backup_name, PRINT_TO_CONSOLE)608 os.system(609 "PGPASSWORD=\"" + CLINLIMS_PWD + "\";export PGPASSWORD; su -c 'pg_dump -h localhost -U clinlims clinlims > " + ROLLBACK_DIR + backup_name + "'")610def backup_config_file():611 app_name = find_installed_name()612 backup_name = get_action_time() + '/' + app_name + '.xml'613 log("Backing up configuration to " + ROLLBACK_DIR + backup_name, PRINT_TO_CONSOLE)614 cmd = 'cp ' + TOMCAT_DIR + '/conf/Catalina/localhost/' + app_name + ".xml " + ROLLBACK_DIR + backup_name615 os.system(cmd)616def do_update():617 log("Updating " + APP_NAME, PRINT_TO_CONSOLE)618 if not find_password(APP_NAME):619 log("Unable to find password from configuration file. Exiting", PRINT_TO_CONSOLE)620 return621 stop_tomcat()622 backup_db()623 backup_war_file()624 backup_config_file()625 backup_plugins()626 delete_app_directory_from_tomcat(APP_NAME)627 delete_work_directory_from_tomcat(APP_NAME)628 copy_war_file_to_tomcat()629 update_with_liquibase()630 # apply_separate_test_fix() -- no longer needed 2.4 or later631 install_backup()632 start_tomcat()633 time.sleep(10)634 if restore_plugins():635 restart_tomcat()636 log("Finished updating " + APP_NAME, PRINT_TO_CONSOLE)637def update():638 if not check_update_preconditions():639 log(APP_NAME + "is not an existing installation, can not update.\n", PRINT_TO_CONSOLE)640 webapps = glob.glob(TOMCAT_DIR + '/webapps/*.war')641 clean_exit()642 do_update()643 # check if backup exists644 if not os.path.exists(BACKUP_DIR + BACKUP_SCRIPT_NAME):645 install_backup()646def ensure_dir_exists(dir):647 if not os.path.exists(dir):648 os.mkdir(dir)649def rollback():650 log("\nRolling back to previous installation", PRINT_TO_CONSOLE)651 if not os.path.exists(ROLLBACK_DIR + get_action_time() + '/' + APP_NAME + '.war'):652 if os.path.exists(TOMCAT_DIR + '/webapps/' + APP_NAME + '.war'):653 log("Application still installed, nothing more to do.", PRINT_TO_CONSOLE)654 else:655 log("Unable to restore application. File missing " + ROLLBACK_DIR + APP_NAME + '.war', PRINT_TO_CONSOLE)656 else:657 cmd = 'cp ' + ROLLBACK_DIR + APP_NAME + '.war ' + TOMCAT_DIR + '/webapps/'658 os.system(cmd)659 log("\nRollback finished", PRINT_TO_CONSOLE)660 log("Please report errors to http://groups.google.com/group/openelisglobal", PRINT_TO_CONSOLE)661 log("or https://openelis.cirg.washington.edu/OpenElisInfo/#contact \n", PRINT_TO_CONSOLE)662 start_tomcat()663def check_on_writable_system():664 if not os.access('./', os.W_OK | os.X_OK):665 print("Unable to write to file system. Please copy installer to hard disk")666 print("Exiting")667 exit()668def write_help():669 print """670setup_OpenELIS.py <options>671 This script must be run as sudo or else it will fail due to permission problems.672-install - Installs OpenELIS. Assumes that there is not a partial install673-installNoPrompt - Installs OpenELIS without prompting. Assumes that there is not a partial install674-installBackup - Installs just the backup. Will overwrite any existing backup675-installCrossTabs - Install just the crosstab functionality into Postgress to allow export to work676-update - Updates OpenElis. Checks to insure that the instance being updated is the same as the installed677-uninstall - Removes OpenELIS from the system optionally including the database. Make sure you have the clinlims password written down someplace678-recover - Will try to recover the system if somebody has tried to fix the system manually. It will re-install the war file and reset the database password679-version - The version number of this installer680-help - This screen681 """682def write_version():683 log("\n", PRINT_TO_CONSOLE)684 log("------------------------------------", not PRINT_TO_CONSOLE)685 log("OpenELIS installer Version " + VERSION, PRINT_TO_CONSOLE)686def open_log_file():687 global LOG_FILE688 LOG_FILE = open(LOG_DIR + LOG_FILE_NAME, 'a')689def clean_exit():690 global LOG_FILE691 LOG_FILE.close()692 exit()693def log(message, to_console):694 LOG_FILE.write(message + "\n")695 if to_console:696 print message697# Main entry point698if len(sys.argv) > 1:699 os.putenv("LANG", LANG_NAME)700 check_on_writable_system()701 ensure_dir_exists(STAGING_DIR)702 ensure_dir_exists(LOG_DIR)703 ensure_dir_exists(ROLLBACK_DIR)704 open_log_file()705 write_version()706 arg = sys.argv[1]707 if arg == "-installCrossTabs":708 log("installCrossTabs " + strftime("%a, %d %b %Y %H:%M:%S", gmtime()), not PRINT_TO_CONSOLE)709 install_crosstab()710 clean_exit()711 if arg == "-uninstall":712 log("uninstall " + strftime("%a, %d %b %Y %H:%M:%S", gmtime()), not PRINT_TO_CONSOLE)713 print "This will uninstall OpenELIS from this machine including **ALL** data from database"714 remove = raw_input("Do you want to continue with the uninstall? y/n: ")715 if remove.lower() == 'y':716 do_uninstall()717 clean_exit()718 if arg == "-update":719 log("update " + strftime("%a, %d %b %Y %H:%M:%S", gmtime()), not PRINT_TO_CONSOLE)720 update()721 install_crosstab()722 # find_installation_name()723 # find_password()724 # apply_separate_test_fix()725 clean_exit()726 if arg == "-installBackup":727 log("installBackup " + strftime("%a, %d %b %Y %H:%M:%S", gmtime()), not PRINT_TO_CONSOLE)728 find_installation_name()729 find_password(APP_NAME)730 install_backup()731 clean_exit()732 if arg == "-install":733 log("install " + strftime("%a, %d %b %Y %H:%M:%S", gmtime()), not PRINT_TO_CONSOLE)734 install()735 install_crosstab()736 clean_exit()737 if arg == "-installNoPrompt":738 log("installNoPrompt " + strftime("%a, %d %b %Y %H:%M:%S", gmtime()), not PRINT_TO_CONSOLE)739 install_no_prompt()740 install_crosstab()741 clean_exit()742 if arg == "-recover":743 log("recover " + strftime("%a, %d %b %Y %H:%M:%S", gmtime()), not PRINT_TO_CONSOLE)744 recover_database()745 clean_exit()746 if arg == "-version":747 # version already written748 clean_exit()749 # if all else fails give help750 write_help()751 clean_exit()752# default is help...
benchmark.py
Source:benchmark.py
...91011PRINT_DETAILS = True1213def print_to_console(string):14 if PRINT_DETAILS == False:15 return16 else:17 sys.stdout.write(str(string) + "\n")181920stopwords_link = "https://raw.githubusercontent.com/stopwords-iso/stopwords-ro/master/stopwords-ro.txt"21stopwords = requests.get(stopwords_link).text.split()2223stanza.download('ro')24stz = stanza.Pipeline('ro')2526wn = rwn.RoWordNet()2728with open("dataset.pickle", "rb") as pickleFile:29 db = pickle.load(pickleFile)303132def most_likely_sense(sentence):33 word = sentence["literal"]3435 sense_count = dict()36 for i in range(len(db[word])):37 correct_sense = db[word][i]['correct_synset_id']38 if correct_sense not in sense_count:39 sense_count[correct_sense] = 140 else:41 sense_count[correct_sense] += 14243 max_l = 044 max_sid = []4546 for sid in sense_count:47 if sense_count[sid] > max_l:48 max_l = sense_count[sid]49 max_sid = []50 max_sid.append(sid)51 elif sense_count[sid] == max_l:52 max_sid.append(sid)5354 return max_sid555657def simplified_lesk(sentence):58 if "synsets" not in sentence:59 synset_ids = wn.synsets(sentence['literal'])60 else:61 synset_ids = sentence['synsets'].split(' ')6263 context = set(sentence["sentence"].split())6465 print_to_console("Context is: ")66 print_to_console(context)6768 lens = []6970 for id in synset_ids:71 try:72 if id != "-1":73 print_to_console("Intersection with: ")74 print_to_console(wn.synset(id).definition.split())75 lens.append(len(context.intersection(76 wn.synset(id).definition.split())))77 print_to_console("Intersection length: " + str(lens[-1]))78 except:79 lens.append(-1)8081 max_l = max(lens)82 max_sid = []8384 for i in range(len(lens)):85 if lens[i] == max_l:86 max_sid.append(synset_ids[i])8788 return max_sid899091def parse_and_tokenize(s):92 # print_to_console(s)93 result = list()94 doc = stz(s)95 for _, sentence in enumerate(doc.sentences):96 for token in sentence.tokens:97 for word in token.words:98 if word.upos in ["VERB", "NOUN", "ADV", "ADJ", "PROPN"]:99 result.append(word.lemma)100 # print_to_console(result)101 return result102103104def split_sentence(s):105 l = []106 for w in s.split():107 w = w.lstrip(string.punctuation).rstrip(string.punctuation).lower()108 if w in stopwords:109 continue110 if len(w) == 0:111 continue112 l.append(w)113 return l114115116def simplified_lesk_wordnet(sentence):117 if "synsets" not in sentence:118 synset_ids = wn.synsets(sentence['literal'])119 else:120 synset_ids = sentence['synsets'].split(' ')121122 context = set(split_sentence(sentence["sentence"]))123124 print_to_console("Context is: ")125 print_to_console(context)126127 lens = []128129 for id in synset_ids:130 try:131 print_to_console("")132 print_to_console("Intersection with synset definition: ")133 print_to_console(split_sentence(wn.synset(id).definition))134 current_len = len(context.intersection(135 split_sentence(wn.synset(id).definition)))136 current_len *= current_len137 print_to_console(138 "Intersection length (squared): " + str(current_len))139 for rel in wn.outbound_relations(id):140 if rel[1] == "hypernym":141 id_hypernym = rel[0]142 print_to_console("Intersection with hypernym " +143 str(wn.synset(id_hypernym).literals) + " definition: ")144 print_to_console(split_sentence(145 wn.synset(id_hypernym).definition))146 current_len += len(context.intersection(147 split_sentence(wn.synset(id_hypernym).definition)))148 print_to_console("Intersection length with hypernym: " + str(len(context.intersection(149 split_sentence(wn.synset(id_hypernym).definition)))))150 print_to_console("Total intersection length: " + str(current_len))151 lens.append(current_len)152 except:153 lens.append(-1)154155 max_l = max(lens)156 max_sid = []157158 for i in range(len(lens)):159 if lens[i] == max_l:160 max_sid.append(synset_ids[i])161162 return max_sid163164165combinations = []166167168def get_combinations(synsets, index, partial):169 if index == len(synsets):170 global combinations171 combinations.append(partial)172 return173 for i in range(len(synsets[index])):174 copy = deepcopy(partial)175 copy.append(synsets[index][i])176 get_combinations(synsets, index+1, copy)177178179def gloss_overlap(gloss1, gloss2):180 words1 = split_sentence(gloss1)181 words2 = split_sentence(gloss2)182 score = 0183 i = 0184 while i < len(words1):185 j = 0186 while j < len(words2):187 if words1[i] == words2[j]:188 seq = [words1[i]]189 l = 1190 i2 = i191 j2 = j192 while j2 + 1 < len(words2) and i2 + 1 < len(words1):193 if words1[i2 + 1] == words2[j2 + 1]:194 seq.append(words1[i2 + 1])195 l += 1196 i2 += 1197 j2 += 1198 else:199 break200 for w in seq:201 if w not in stopwords:202 score += l * l203 break204 j += 1205 i += 1206 return score207208209def adapted_lesk(sentence, n=2):210 word = sentence['literal']211 text_prefix = sentence['text_prefix']212 text_postfix = sentence['text_postfix']213214 global combinations215 combinations = []216 context = [word]217 text_prefix = parse_and_tokenize(text_prefix)218 text_prefix.reverse()219 text_postfix = parse_and_tokenize(text_postfix)220 for i in range(max(len(text_prefix), len(text_postfix))):221 for text in [text_prefix, text_postfix]:222 if i < len(text) and len(context) < 2 * n + 1:223 # we include in the context only words from the db224 if(text[i] in db):225 if text[i] not in stopwords:226 context.append(text[i])227 synsets = []228229 print_to_console("Context is: ")230 print_to_console(context)231 for cword in context:232 synsets_cword = db[cword][0]['synsets'].split()[:-1]233 synsets.append(synsets_cword)234 print_to_console("The word " + cword + " appears in " +235 str(len(synsets_cword)) + " synsets")236 get_combinations(synsets, 0, [])237 max_score = 0238 results = []239 print_to_console("There are " + str(len(combinations)) +240 " combinations possible")241 for c in combinations:242 d = []243 score = 0244 for s in c:245 d.append(wn.synset(s).definition)246 print_to_console(247 "We calculate the overlap between the definitions (each pair): ")248 print_to_console(d)249 for d1 in d:250 for d2 in d:251 if d1 != d2:252 score += gloss_overlap(d1, d2)253 print_to_console("Final score is " + str(score))254 print_to_console("")255 if score > max_score:256 max_score = score257 results = []258 results.append(c[0])259 elif score == max_score:260 results.append(c[0])261262 freq_results = dict()263 for r in results:264 if r not in freq_results:265 freq_results[r] = 1266 else:267 freq_results[r] += 1268269 most_freq = []270 max_freq = 0271 for r in freq_results:272 if freq_results[r] > max_freq:273 max_freq = freq_results[r]274 most_freq = []275 most_freq.append(r)276 elif freq_results[r] == max_freq:277 most_freq.append(r)278279 return most_freq280281282possible_rel = set()283284285def simplified_lesk_wordnet_complete(sentence):286 if "synsets" not in sentence:287 synset_ids = wn.synsets(sentence['literal'])288 else:289 synset_ids = sentence['synsets'].split(' ')290291 context = set(split_sentence(sentence["sentence"]))292293 print_to_console("Context is: ")294 print_to_console(context)295296 lens = []297298 for id in synset_ids:299 try:300 print_to_console("")301 print_to_console("Intersection with synset definition: ")302 print_to_console(split_sentence(wn.synset(id).definition))303 current_len = len(context.intersection(304 split_sentence(wn.synset(id).definition)))305 current_len *= current_len306 print_to_console(307 "Intersection length (squared): " + str(current_len))308 for rel in wn.outbound_relations(id):309 if rel[1] not in ["hyponym", "instance_hyponym", "substance_meronym", "member_meronym", "part_meronym"]:310 id_hypernym = rel[0]311 print_to_console("Intersection with " + rel[1] + " " +312 str(wn.synset(id_hypernym).literals) + " definition: ")313 print_to_console(split_sentence(314 wn.synset(id_hypernym).definition))315 current_len += len(context.intersection(316 split_sentence(wn.synset(id_hypernym).definition)))317 print_to_console("Intersection length with hypernym: " + str(len(context.intersection(318 split_sentence(wn.synset(id_hypernym).definition)))))319 print_to_console("Total intersection length: " + str(current_len))320 lens.append(current_len)321 except:322 lens.append(-1)323324 max_l = max(lens)325 max_sid = []326327 for i in range(len(lens)):328 if lens[i] == max_l:329 max_sid.append(synset_ids[i])330331 return max_sid332333334last_word = None335last_definitions = None336337338def parsed_simplified_lesk(sentence):339 if "synsets" not in sentence:340 synset_ids = wn.synsets(sentence['literal'])341 else:342 synset_ids = sentence['synsets'].split(' ')343344 context = set(parse_and_tokenize(sentence["sentence"]))345 print_to_console("Context is: ")346 print_to_console(context)347348 global last_word349 global last_definitions350 if last_word == sentence['literal']:351 definitions = last_definitions352 else:353 definitions = []354 for id in synset_ids:355 try:356 definition = set(parse_and_tokenize(wn.synset(id).definition))357 definitions.append(definition)358 except:359 continue360 last_definitions = definitions361 last_word = sentence['literal']362363 lens = []364365 for definition in definitions:366 try:367 # print_to_console(context, definition)368 print_to_console("Intersection with: ")369 print_to_console(definition)370 lens.append(len(context.intersection(definition)))371 print_to_console("Intersection length: " +372 str(len(context.intersection(definition))))373 except:374 lens.append(-1)375376 # print_to_console(lens)377 max_l = max(lens)378 max_sid = []379380 for i in range(len(lens)):381 if lens[i] == max_l:382 max_sid.append(synset_ids[i])383384 return max_sid385386387def benchmark(f, limit=None, word_list=None):388 print_to_console("----- Testing " + f.__name__ + " -----")389390 correct_total_s = 0391 correct_total_l = 0392 computed_sentences_total = 0393394 filename_strict = "output/" + f.__name__ + "_strict"395 filename_loose = "output/" + f.__name__ + "_loose"396 filename_csv = "output/" + f.__name__397398 if word_list is not None or limit is not None:399 filename_loose += ".partial"400 filename_strict += ".partial"401 filename_csv += ".partial"402403 out_strict = open(filename_strict + ".txt", "wt", encoding="utf-8")404 csv_out = open(filename_csv + ".csv",405 "w", encoding="utf-8", newline='')406 out_loose = open(filename_loose + ".txt", "wt", encoding="utf-8")407 header = ['word', 'sentence_id', 'correct_synset_id',408 'predicted_synsets', 'chosen_synset']409 writer = csv.writer(csv_out)410 writer.writerow(header)411 word_count = 0412413 if word_list is None:414 word_list = db415416 for word in word_list:417 word_sentence_nr = len(db[word])418 correct_s = 0419 correct_l = 0420 computed_sentences = 0421 for i in range(word_sentence_nr):422 correct_sid = db[word][i]['correct_synset_id']423 if correct_sid == '-1':424 continue425426 computed_sentences = computed_sentences + 1427 result = f(db[word][i])428 result_str = ""429 for r in result:430 result_str += r + " "431 if len(result) == 1:432 writer.writerow(433 [word, i, correct_sid, result_str[:-1], result[0]])434 if result[0] == correct_sid:435 correct_s += 1436 else:437 max_freq = -1438 max_sid = result[0]439 for sid in result:440 freq = 0441 for k in range(len(db[word])):442 if db[word][k]['correct_synset_id'] == sid:443 freq += 1444 if freq > max_freq:445 max_freq = freq446 max_sid = sid447 writer.writerow(448 [word, i, correct_sid, result_str[:-1], max_sid])449 if max_sid == correct_sid:450 correct_l += 1451452 # print('Correct strict guesses for word ' + word + ": " +453 # str(correct_s) + "/" + str(computed_sentences))454 # print('Correct loose guesses for word ' + word + ": " +455 # str(correct_l) + "/" + str(computed_sentences))456 out_strict.write('Correct guesses for word ' + word + ": " +457 str(correct_s) + "/" + str(computed_sentences) + "\n")458 correct_total_s += correct_s459 out_loose.write('Correct guesses for word ' + word + ": " +460 str(correct_l) + "/" + str(computed_sentences) + "\n")461 correct_total_l += correct_l + correct_s462 computed_sentences_total += computed_sentences463 word_count += 1464 if limit and word_count == limit:465 break466467 out_strict.write('Correct guesses: ' + str(correct_total_s) +468 "/" + str(computed_sentences_total))469 out_loose.write('Correct guesses: ' + str(correct_total_l) +470 "/" + str(computed_sentences_total))471472 print('Correct strict guesses: ' + str(correct_total_s) +473 "/" + str(computed_sentences_total))474 print('Correct loose guesses: ' + str(correct_total_l) +475 "/" + str(computed_sentences_total))476477 out_strict.close()478 out_loose.close()479 csv_out.close()480481482def main():483 # benchmark(simplified_lesk)484 # benchmark(parsed_simplified_lesk, limit=1)485 # benchmark(parsed_simplified_lesk, word_list=["rol"])486 # benchmark(simplified_lesk_wordnet)487 # benchmark(simplified_lesk_wordnet_complete)488 # benchmark(adapted_lesk, word_list=["mÄr"])489 # benchmark(most_likely_sense)490491 # sentence = {492 # "sentence": "MÄnânc un mÄr roÈu cules din pom.", "literal": "mÄr"}493 # result_id = parsed_simplified_lesk(sentence)494 # wn.print_synset(result_id)495496 word = "bancÄ"497 print_to_console("CÄutÄm înÈelesul cuvântului " + word +498 " în propoziÈia: " + db[word][0]['sentence'])499 result = simplified_lesk(db[word][0])500501 print_to_console("Selected definitions:")502 for id in result:503 print_to_console(wn.synset(id).definition)504 print_to_console("Definition marked as correct: ")505 print_to_console(wn.synset(db[word][0]['correct_synset_id']).definition)506507 print_to_console("___________________________________")508509 print_to_console("CÄutÄm înÈelesul cuvântului " + word +510 " în propoziÈia: " + db[word][0]['sentence'])511 result = parsed_simplified_lesk(db[word][0])512513 print_to_console("Selected definitions: ")514 for id in result:515 print_to_console(wn.synset(id).definition)516 print_to_console("Definition marked as correct: ")517 print_to_console(wn.synset(db[word][0]['correct_synset_id']).definition)518519 print_to_console("___________________________________")520521 print_to_console("CÄutÄm înÈelesul cuvântului " + word +522 " în propoziÈia: " + db[word][0]['sentence'])523 result = simplified_lesk_wordnet(db[word][0])524525 print_to_console("Selected definitions:")526 for id in result:527 print_to_console(wn.synset(id).definition)528 print_to_console("Definition marked as correct:")529 print_to_console(wn.synset(db[word][0]['correct_synset_id']).definition)530531 print_to_console("___________________________________")532533 print_to_console("CÄutÄm înÈelesul cuvântului " + word +534 " în propoziÈia: " + db[word][0]['sentence'])535 result = simplified_lesk_wordnet_complete(db[word][0])536537 print_to_console("Selected definitions:")538 for id in result:539 print_to_console(wn.synset(id).definition)540 print_to_console("Definition marked as correct:")541 print_to_console(wn.synset(db[word][0]['correct_synset_id']).definition)542543 # print_to_console("___________________________________")544545 # print_to_console("CÄutÄm înÈelesul cuvântului " + word +546 # " în propoziÈia: " + db[word][0]['sentence'])547 # result = adapted_lesk(db[word][0])548549 # print_to_console("Selected definitions:")550 # for id in result:551 # print_to_console(wn.synset(id).definition)552 # print_to_console("Definition marked as correct:")553 # print_to_console(wn.synset(db[word][0]['correct_synset_id']).definition)554555 return556557558if __name__ == "__main__":
...
venusverifier.py
Source:venusverifier.py
1import multiprocessing as mp2from multiprocessing import Process3import queue4from timeit import default_timer as timer5from src.layersmodel import LayersModel6from src.splittingprocess import SplittingProcess7class WorkerVerificationProcess(Process):8 TIMEOUT = 36009 def __init__(self, id, jobs_queue, reporting_queue, print=False):10 super(WorkerVerificationProcess, self).__init__()11 self.id = id12 self.jobs_queue = jobs_queue13 self.reporting_queue = reporting_queue14 self.PRINT_TO_CONSOLE = print15 def run(self):16 while True:17 try:18 job_id, vmodel = self.jobs_queue.get(timeout=self.TIMEOUT)19 start = timer()20 vmodel.encode()21 (res, gap) = vmodel.verify()22 end = timer()23 runtime = end - start24 if self.PRINT_TO_CONSOLE:25 print("Subprocess", self.id, "finished job", job_id, "result:", res, "in", runtime)26 self.reporting_queue.put((job_id, res, runtime, gap))27 except queue.Empty:28 # to handle the case when the main process got killed,29 # but the workers remained alive.30 # the worker waits for at most 2 minutes to get a new job,31 # if no job was found, it terminates32 break33class VenusVerifier:34 def __init__(self, nmodel, spec, encoder_params, splitting_params, print_to_console=False):35 super(VenusVerifier, self).__init__()36 self.TIME_LIMIT = encoder_params.TIME_LIMIT37 self.PARALLEL_PROCESSES_NUMBER = encoder_params.WORKERS_NUMBER38 # convert keras nmodel into our internal representation LayersModel39 lmodel = LayersModel(encoder_params.ENCODING)40 lmodel.load(nmodel, spec)41 # the queue to which all worker processes report the results42 # and the splitting process will store the total number of splits43 self.reporting_queue = mp.Queue()44 jobs_queue = mp.Queue()45 # compute the initial splits46 aux_splitting_process = SplittingProcess(0, lmodel, spec, splitting_params, encoder_params,47 jobs_queue, self.reporting_queue,48 print_to_console)49 initial_splits = aux_splitting_process.get_initial_splits()50 # start a splitting process for each initial split51 self.splitting_processes = [SplittingProcess(i+1, initial_splits[i][0], initial_splits[i][1],52 splitting_params, encoder_params,53 jobs_queue, self.reporting_queue,54 print_to_console)55 for i in range(len(initial_splits))]56 self.worker_processes = [WorkerVerificationProcess(i+1, jobs_queue, self.reporting_queue, print_to_console)57 for i in range(self.PARALLEL_PROCESSES_NUMBER)]58 self.PRINT_TO_CONSOLE = print_to_console59 def verify(self):60 start = timer()61 # start the splitting and worker processes62 for proc in self.splitting_processes:63 proc.start()64 for proc in self.worker_processes:65 proc.start()66 timedout_jobs_count = 067 finished_jobs_count = 068 finished_splitting_processes_count = 069 result = None70 total_number_of_splits = -171 """ 72 Read results from the reporting queue73 until encountered a True result, or74 until all the splits have completed75 """76 while True:77 try:78 job_id, res, runtime, extra = self.reporting_queue.get(timeout=self.TIME_LIMIT - (timer() - start))79 if res == True:80 if self.PRINT_TO_CONSOLE:81 print("Main process: read True. Terminating...")82 result = ("True", "{}".format(finished_jobs_count+1), extra)83 break84 elif res == False:85 finished_jobs_count += 186 elif res == "Timeout":87 finished_jobs_count += 188 timedout_jobs_count += 189 elif res == SplittingProcess.TOTAL_JOBS_NUMBER_STRING:90 # update the total_number of splits91 if total_number_of_splits == -1:92 total_number_of_splits = job_id93 else:94 total_number_of_splits += job_id95 finished_splitting_processes_count += 196 else:97 raise Exception("Unexpected result read from reporting queue", res)98 # stopping conditions99 if total_number_of_splits != -1 and finished_splitting_processes_count == len(self.splitting_processes) and\100 finished_jobs_count >= total_number_of_splits:101 if self.PRINT_TO_CONSOLE:102 print("Main process: all subproblems have finished. Terminating...")103 if timedout_jobs_count == 0:104 result = ("False", total_number_of_splits, None)105 else:106 result = ("Timeout", total_number_of_splits, None)107 break108 except queue.Empty:109 # Timout occured110 result = ("Timeout", finished_jobs_count, None)111 break112 except KeyboardInterrupt:113 # Received terminating signal114 result = ("Interrupted", finished_jobs_count, None)115 break116 """117 Terminate the splitting and worker processes.118 Especially relevant if there was one early True result.119 """120 try:121 for proc in self.splitting_processes:122 proc.terminate()123 for proc in self.worker_processes:124 proc.terminate()125 except:126 print("Error while attempting to terminate processes")...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run grail automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
