Initial commit
commit
4aa07b7942
GPG Key ID:
8CAB39FC9A8F13FB
@ -0,0 +1,248 @@
|
||||
|
||||
# Created by https://www.toptal.com/developers/gitignore/api/pycharm,python
|
||||
# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm,python
|
||||
|
||||
### PyCharm ###
|
||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
|
||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||
|
||||
# User-specific stuff
|
||||
.idea/**/workspace.xml
|
||||
.idea/**/tasks.xml
|
||||
.idea/**/usage.statistics.xml
|
||||
.idea/**/dictionaries
|
||||
.idea/**/shelf
|
||||
|
||||
# Generated files
|
||||
.idea/**/contentModel.xml
|
||||
|
||||
# Sensitive or high-churn files
|
||||
.idea/**/dataSources/
|
||||
.idea/**/dataSources.ids
|
||||
.idea/**/dataSources.local.xml
|
||||
.idea/**/sqlDataSources.xml
|
||||
.idea/**/dynamic.xml
|
||||
.idea/**/uiDesigner.xml
|
||||
.idea/**/dbnavigator.xml
|
||||
|
||||
# Gradle
|
||||
.idea/**/gradle.xml
|
||||
.idea/**/libraries
|
||||
|
||||
# Gradle and Maven with auto-import
|
||||
# When using Gradle or Maven with auto-import, you should exclude module files,
|
||||
# since they will be recreated, and may cause churn. Uncomment if using
|
||||
# auto-import.
|
||||
# .idea/artifacts
|
||||
# .idea/compiler.xml
|
||||
# .idea/jarRepositories.xml
|
||||
# .idea/modules.xml
|
||||
# .idea/*.iml
|
||||
# .idea/modules
|
||||
# *.iml
|
||||
# *.ipr
|
||||
|
||||
# CMake
|
||||
cmake-build-*/
|
||||
|
||||
# Mongo Explorer plugin
|
||||
.idea/**/mongoSettings.xml
|
||||
|
||||
# File-based project format
|
||||
*.iws
|
||||
|
||||
# IntelliJ
|
||||
out/
|
||||
|
||||
# mpeltonen/sbt-idea plugin
|
||||
.idea_modules/
|
||||
|
||||
# JIRA plugin
|
||||
atlassian-ide-plugin.xml
|
||||
|
||||
# Cursive Clojure plugin
|
||||
.idea/replstate.xml
|
||||
|
||||
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||
com_crashlytics_export_strings.xml
|
||||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
fabric.properties
|
||||
|
||||
# Editor-based Rest Client
|
||||
.idea/httpRequests
|
||||
|
||||
# Android studio 3.1+ serialized cache file
|
||||
.idea/caches/build_file_checksums.ser
|
||||
|
||||
### PyCharm Patch ###
|
||||
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
|
||||
|
||||
# *.iml
|
||||
# modules.xml
|
||||
# .idea/misc.xml
|
||||
# *.ipr
|
||||
|
||||
# Sonarlint plugin
|
||||
# https://plugins.jetbrains.com/plugin/7973-sonarlint
|
||||
.idea/**/sonarlint/
|
||||
|
||||
# SonarQube Plugin
|
||||
# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
|
||||
.idea/**/sonarIssues.xml
|
||||
|
||||
# Markdown Navigator plugin
|
||||
# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
|
||||
.idea/**/markdown-navigator.xml
|
||||
.idea/**/markdown-navigator-enh.xml
|
||||
.idea/**/markdown-navigator/
|
||||
|
||||
# Cache file creation bug
|
||||
# See https://youtrack.jetbrains.com/issue/JBR-2257
|
||||
.idea/$CACHE_FILE$
|
||||
|
||||
# CodeStream plugin
|
||||
# https://plugins.jetbrains.com/plugin/12206-codestream
|
||||
.idea/codestream.xml
|
||||
|
||||
### Python ###
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
pytestdebug.log
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
doc/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
pythonenv*
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# profiling data
|
||||
.prof
|
||||
|
||||
# End of https://www.toptal.com/developers/gitignore/api/pycharm,python
|
||||
@ -0,0 +1,8 @@
|
||||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# Datasource local storage ignored files
|
||||
/../../../../../../:\Users\jdc10\PycharmProjects\NikeBot\.idea/dataSources/
|
||||
/dataSources.local.xml
|
||||
# Editor-based HTTP Client requests
|
||||
/httpRequests/
|
||||
@ -0,0 +1,11 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="PYTHON_MODULE" version="4">
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/venv" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
</module>
|
||||
@ -0,0 +1,98 @@
|
||||
<component name="InspectionProjectProfileManager">
|
||||
<profile version="1.0">
|
||||
<option name="myName" value="Project Default" />
|
||||
<inspection_tool class="JsonStandardCompliance" enabled="false" level="ERROR" enabled_by_default="false" />
|
||||
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
|
||||
<option name="ignoredPackages">
|
||||
<value>
|
||||
<list size="70">
|
||||
<item index="0" class="java.lang.String" itemvalue="traitlets" />
|
||||
<item index="1" class="java.lang.String" itemvalue="pywinpty" />
|
||||
<item index="2" class="java.lang.String" itemvalue="typed-ast" />
|
||||
<item index="3" class="java.lang.String" itemvalue="testpath" />
|
||||
<item index="4" class="java.lang.String" itemvalue="pickleshare" />
|
||||
<item index="5" class="java.lang.String" itemvalue="defusedxml" />
|
||||
<item index="6" class="java.lang.String" itemvalue="QtPy" />
|
||||
<item index="7" class="java.lang.String" itemvalue="astroid" />
|
||||
<item index="8" class="java.lang.String" itemvalue="ipython-genutils" />
|
||||
<item index="9" class="java.lang.String" itemvalue="appdirs" />
|
||||
<item index="10" class="java.lang.String" itemvalue="elasticsearch" />
|
||||
<item index="11" class="java.lang.String" itemvalue="Faker" />
|
||||
<item index="12" class="java.lang.String" itemvalue="Pygments" />
|
||||
<item index="13" class="java.lang.String" itemvalue="mccabe" />
|
||||
<item index="14" class="java.lang.String" itemvalue="pyzmq" />
|
||||
<item index="15" class="java.lang.String" itemvalue="bleach" />
|
||||
<item index="16" class="java.lang.String" itemvalue="certifi" />
|
||||
<item index="17" class="java.lang.String" itemvalue="entrypoints" />
|
||||
<item index="18" class="java.lang.String" itemvalue="pyparsing" />
|
||||
<item index="19" class="java.lang.String" itemvalue="jsonschema" />
|
||||
<item index="20" class="java.lang.String" itemvalue="notebook" />
|
||||
<item index="21" class="java.lang.String" itemvalue="pywin32" />
|
||||
<item index="22" class="java.lang.String" itemvalue="qtconsole" />
|
||||
<item index="23" class="java.lang.String" itemvalue="terminado" />
|
||||
<item index="24" class="java.lang.String" itemvalue="wrapt" />
|
||||
<item index="25" class="java.lang.String" itemvalue="jupyter-client" />
|
||||
<item index="26" class="java.lang.String" itemvalue="ipykernel" />
|
||||
<item index="27" class="java.lang.String" itemvalue="nbconvert" />
|
||||
<item index="28" class="java.lang.String" itemvalue="attrs" />
|
||||
<item index="29" class="java.lang.String" itemvalue="jedi" />
|
||||
<item index="30" class="java.lang.String" itemvalue="regex" />
|
||||
<item index="31" class="java.lang.String" itemvalue="backcall" />
|
||||
<item index="32" class="java.lang.String" itemvalue="widgetsnbextension" />
|
||||
<item index="33" class="java.lang.String" itemvalue="elasticsearch-dsl" />
|
||||
<item index="34" class="java.lang.String" itemvalue="pathspec" />
|
||||
<item index="35" class="java.lang.String" itemvalue="decorator" />
|
||||
<item index="36" class="java.lang.String" itemvalue="jupyter-core" />
|
||||
<item index="37" class="java.lang.String" itemvalue="pandocfilters" />
|
||||
<item index="38" class="java.lang.String" itemvalue="wcwidth" />
|
||||
<item index="39" class="java.lang.String" itemvalue="numpy" />
|
||||
<item index="40" class="java.lang.String" itemvalue="importlib-metadata" />
|
||||
<item index="41" class="java.lang.String" itemvalue="pyrsistent" />
|
||||
<item index="42" class="java.lang.String" itemvalue="jupyter" />
|
||||
<item index="43" class="java.lang.String" itemvalue="zipp" />
|
||||
<item index="44" class="java.lang.String" itemvalue="prompt-toolkit" />
|
||||
<item index="45" class="java.lang.String" itemvalue="ipywidgets" />
|
||||
<item index="46" class="java.lang.String" itemvalue="six" />
|
||||
<item index="47" class="java.lang.String" itemvalue="tornado" />
|
||||
<item index="48" class="java.lang.String" itemvalue="parso" />
|
||||
<item index="49" class="java.lang.String" itemvalue="text-unidecode" />
|
||||
<item index="50" class="java.lang.String" itemvalue="black" />
|
||||
<item index="51" class="java.lang.String" itemvalue="nbformat" />
|
||||
<item index="52" class="java.lang.String" itemvalue="ipython" />
|
||||
<item index="53" class="java.lang.String" itemvalue="Send2Trash" />
|
||||
<item index="54" class="java.lang.String" itemvalue="packaging" />
|
||||
<item index="55" class="java.lang.String" itemvalue="toml" />
|
||||
<item index="56" class="java.lang.String" itemvalue="prometheus-client" />
|
||||
<item index="57" class="java.lang.String" itemvalue="mistune" />
|
||||
<item index="58" class="java.lang.String" itemvalue="pandas" />
|
||||
<item index="59" class="java.lang.String" itemvalue="lazy-object-proxy" />
|
||||
<item index="60" class="java.lang.String" itemvalue="colorama" />
|
||||
<item index="61" class="java.lang.String" itemvalue="pylint" />
|
||||
<item index="62" class="java.lang.String" itemvalue="jupyter-console" />
|
||||
<item index="63" class="java.lang.String" itemvalue="pyzipcode" />
|
||||
<item index="64" class="java.lang.String" itemvalue="isort" />
|
||||
<item index="65" class="java.lang.String" itemvalue="webencodings" />
|
||||
<item index="66" class="java.lang.String" itemvalue="discord" />
|
||||
<item index="67" class="java.lang.String" itemvalue="aiohttp" />
|
||||
<item index="68" class="java.lang.String" itemvalue="multidict" />
|
||||
<item index="69" class="java.lang.String" itemvalue="yarl" />
|
||||
</list>
|
||||
</value>
|
||||
</option>
|
||||
</inspection_tool>
|
||||
<inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
|
||||
<option name="ignoredErrors">
|
||||
<list>
|
||||
<option value="E131" />
|
||||
</list>
|
||||
</option>
|
||||
</inspection_tool>
|
||||
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
|
||||
<option name="ignoredErrors">
|
||||
<list>
|
||||
<option value="N802" />
|
||||
</list>
|
||||
</option>
|
||||
</inspection_tool>
|
||||
</profile>
|
||||
</component>
|
||||
@ -0,0 +1,6 @@
|
||||
<component name="InspectionProjectProfileManager">
|
||||
<settings>
|
||||
<option name="USE_PROJECT_PROFILE" value="false" />
|
||||
<version value="1.0" />
|
||||
</settings>
|
||||
</component>
|
||||
@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (NikeBot)" project-jdk-type="Python SDK" />
|
||||
</project>
|
||||
@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/NikeBot.iml" filepath="$PROJECT_DIR$/.idea/NikeBot.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,159 @@
|
||||
import logging
|
||||
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver import Chrome, ChromeOptions
|
||||
|
||||
# from msedge.selenium_tools.options import Options
|
||||
# from msedge.selenium_tools import Edge
|
||||
from selenium.common.exceptions import NoSuchElementException
|
||||
import os
|
||||
import pickle
|
||||
import time
|
||||
from colored import fg, attr
|
||||
from uuid import uuid4
|
||||
from random import randrange, choice
|
||||
|
||||
worker = None
|
||||
|
||||
|
||||
class Worker(object):
|
||||
def __init__(
|
||||
self, username, password, cookie_name, port, headless=False, proxy=True
|
||||
):
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.cookie_name = cookie_name
|
||||
self.headless = headless
|
||||
self.driver = self.set_driver(headless, proxy, port)
|
||||
self.uuid = uuid4()
|
||||
self.fg = randrange(1, 7)
|
||||
self.attr = choice([0, 1, 4])
|
||||
|
||||
def set_driver(self, headless, proxy, port) -> webdriver.Edge:
|
||||
logging.info(port)
|
||||
if os.path.isfile("webdrivers/chromedriver.exe"):
|
||||
chrome_driver = "webdrivers/chromedriver.exe"
|
||||
else:
|
||||
print("No driver found.")
|
||||
raise FileNotFoundError
|
||||
options = ChromeOptions()
|
||||
# options.add_argument('ignore-certificate-errors')
|
||||
options.add_argument("disable-web-security")
|
||||
# options.add_extension("data/webrtc.crx")
|
||||
|
||||
if headless:
|
||||
options.headless = True
|
||||
|
||||
capabilities = webdriver.DesiredCapabilities.CHROME.copy()
|
||||
if proxy:
|
||||
capabilities["proxy"] = {
|
||||
"httpProxy": f"192.168.1.52:{port}",
|
||||
"ftpProxy": f"192.168.1.52:{port}",
|
||||
"sslProxy": f"192.168.1.52:{port}",
|
||||
"proxyType": "MANUAL",
|
||||
}
|
||||
else:
|
||||
logging.warning("Not using proxy!")
|
||||
|
||||
return Chrome(
|
||||
options=options,
|
||||
executable_path=chrome_driver,
|
||||
desired_capabilities=capabilities,
|
||||
)
|
||||
|
||||
def login(self):
|
||||
driver = self.driver
|
||||
driver.implicitly_wait(5)
|
||||
driver.get("https://www.nike.com/login")
|
||||
driver.find_element_by_css_selector("input[autocomplete='email']").send_keys(self.username)
|
||||
driver.find_element_by_css_selector('input[autocomplete="current-password"]').send_keys(self.password)
|
||||
driver.find_element_by_css_selector('input[value="SIGN IN"]').click()
|
||||
|
||||
success = False
|
||||
while not success:
|
||||
time.sleep(2)
|
||||
|
||||
try:
|
||||
driver.implicitly_wait(1)
|
||||
driver.find_element_by_css_selector('input[value="Dismiss this error"]').click()
|
||||
driver.implicitly_wait(5)
|
||||
driver.find_element_by_css_selector('input[autocomplete="current-password"]').send_keys(self.password)
|
||||
driver.find_element_by_css_selector('input[value="SIGN IN"]').click()
|
||||
except:
|
||||
success = True
|
||||
# if driver.find_element_by_css_selector()
|
||||
|
||||
def go(self):
|
||||
driver = self.driver
|
||||
driver.implicitly_wait(2)
|
||||
|
||||
logging.info(f"Starting on thing")
|
||||
driver.get("http://lumtest.com/myip.json")
|
||||
time.sleep(100)
|
||||
|
||||
logging.error("Uh oh, I made it too far!")
|
||||
|
||||
def close(self):
|
||||
self.driver.close()
|
||||
self.driver.quit()
|
||||
|
||||
@staticmethod
|
||||
def get_node_text(webElement):
|
||||
text: str = webElement.text
|
||||
for child in webElement.find_elements_by_xpath("./*"):
|
||||
text = text.replace(child.text, "")
|
||||
return text
|
||||
|
||||
def print(self, string: str):
|
||||
print(
|
||||
f"{fg(self.fg)}{str(self.uuid)[:6]}--{string}{attr(self.attr)}", flush=True
|
||||
)
|
||||
|
||||
def check_for_error(self, webElement):
|
||||
pass
|
||||
|
||||
|
||||
def initializer(login_list, port_list, log_queue, log_config, level=logging.DEBUG):
|
||||
log_config(log_queue, level)
|
||||
logging.info("init-start")
|
||||
global worker
|
||||
port = port_list.pop()
|
||||
login_info = login_list.pop()
|
||||
worker = Worker(
|
||||
login_info["un"],
|
||||
login_info["pw"],
|
||||
login_info["name"],
|
||||
port,
|
||||
headless=False,
|
||||
proxy=True,
|
||||
)
|
||||
worker.login()
|
||||
logging.info("init-fin")
|
||||
|
||||
|
||||
def start():
|
||||
global worker
|
||||
result = worker.go()
|
||||
if result:
|
||||
logging.info(f"Finished successfully! {result}")
|
||||
else:
|
||||
logging.warning(f"Failed! {result}")
|
||||
return result
|
||||
|
||||
|
||||
def before_exit():
|
||||
global worker
|
||||
worker.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
username = os.getenv("USERNAME")
|
||||
password = os.getenv("PASSWORD")
|
||||
|
||||
cdriver = Worker(username, password, headless=False)
|
||||
cdriver.login()
|
||||
|
||||
cdriver.go("Tyler", "Menezes", "Washington")
|
||||
cdriver.go("Jacob", "Cover", "San Diego")
|
||||
cdriver.go("TJ", "Horner", "San Diego")
|
||||
cdriver.close()
|
||||
@ -0,0 +1,58 @@
|
||||
# From https://docs.python.org/3.7/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes
|
||||
|
||||
import logging
|
||||
import logging.handlers
|
||||
import multiprocessing
|
||||
|
||||
|
||||
def listener_configurer(console_level):
|
||||
root = logging.getLogger()
|
||||
rfh = logging.handlers.RotatingFileHandler("../linkedout.log", "a", 1500000, 10)
|
||||
rfh_f = logging.Formatter(
|
||||
"%(asctime)s | %(processName)-10s | %(name)s | %(levelname)-8s | %(message)s"
|
||||
)
|
||||
rfh.setFormatter(rfh_f)
|
||||
root.addHandler(rfh)
|
||||
sh = logging.StreamHandler()
|
||||
sh.setLevel(console_level)
|
||||
sh_f = logging.Formatter(
|
||||
"%(processName)-10s %(levelname)-8s | %(message)s"
|
||||
)
|
||||
sh.setFormatter(sh_f)
|
||||
root.addHandler(sh)
|
||||
root.info("Starting logging")
|
||||
|
||||
|
||||
# This is the listener process top-level loop: wait for logging events
|
||||
# (LogRecords)on the queue and handle them, quit when you get a None for a
|
||||
# LogRecord.
|
||||
def listener_process(queue, configurer, console_level=logging.INFO):
|
||||
configurer(console_level)
|
||||
while True:
|
||||
try:
|
||||
record = queue.get()
|
||||
if (
|
||||
record is None
|
||||
): # We send this as a sentinel to tell the listener to quit.
|
||||
break
|
||||
logger = logging.getLogger(record.name)
|
||||
logger.handle(record) # No level or filter logic applied - just do it!
|
||||
except Exception:
|
||||
import sys, traceback
|
||||
|
||||
print("Whoops! Problem:", file=sys.stderr)
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
|
||||
|
||||
LEVELS = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL]
|
||||
|
||||
|
||||
# The worker configuration is done at the start of the worker process run.
|
||||
# Note that on Windows you can't rely on fork semantics, so each process
|
||||
# will run the logging configuration code when it starts.
|
||||
def worker_configurer(queue, level=logging.DEBUG):
|
||||
h = logging.handlers.QueueHandler(queue) # Just the one handler needed
|
||||
root = logging.getLogger()
|
||||
root.addHandler(h)
|
||||
# send all messages, for demo; no other level or filter logic applied.
|
||||
root.setLevel(level)
|
||||
@ -0,0 +1,47 @@
|
||||
import os
|
||||
|
||||
from src import Worker
|
||||
from os import getenv
|
||||
from multiprocessing import Pool, Queue, Process, Manager
|
||||
import csv
|
||||
import json
|
||||
from src.logger import listener_process, listener_configurer, worker_configurer
|
||||
|
||||
|
||||
def setup_logins_ports():
|
||||
manager = Manager()
|
||||
shared_ip_list = manager.list([24000, 24000, 24000, 24001, 24001, 24001])
|
||||
login_list = manager.list([
|
||||
{"name": "johnpeter", "un": getenv("USERNAME1"), "pw": getenv("PASSWORD1")},
|
||||
{"name": "johnpeter", "un": getenv("USERNAME1"), "pw": getenv("PASSWORD1")},
|
||||
{"name": "johnpeter", "un": getenv("USERNAME1"), "pw": getenv("PASSWORD1")},
|
||||
{"name": "peterjohn", "un": getenv("USERNAME2"), "pw": getenv("PASSWORD2")},
|
||||
{"name": "peterjohn", "un": getenv("USERNAME2"), "pw": getenv("PASSWORD2")},
|
||||
{"name": "peterjohn", "un": getenv("USERNAME2"), "pw": getenv("PASSWORD2")},
|
||||
])
|
||||
return shared_ip_list, login_list
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
num_processes = int(getenv("NUM_PROCESSES", 6))
|
||||
num_total_processes = int(getenv("NUM_PROCESSES", 1))
|
||||
|
||||
with open("data/results", "w") as results_file:
|
||||
# Start the data at the last known chunk, so if something fails, only a little progress is lost
|
||||
# Setup some logging things
|
||||
log_queue = Queue(-1)
|
||||
log_listener = Process(target=listener_process, args=(log_queue, listener_configurer))
|
||||
log_listener.start()
|
||||
|
||||
# Loop through, one chunk at a time
|
||||
for i in range(num_total_processes):
|
||||
# Setup the list of ports and login information
|
||||
shared_ip_list, login_list = setup_logins_ports()
|
||||
with Pool(num_processes, initializer=Worker.initializer,
|
||||
initargs=(login_list, shared_ip_list, log_queue, worker_configurer)) as p:
|
||||
result = p.apply(Worker.start)
|
||||
for i in range(num_processes):
|
||||
p.apply(Worker.before_exit)
|
||||
results_file.flush()
|
||||
results_file.write(f"{json.dumps(result)} \n")
|
||||
results_file.flush()
|
||||
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue