commit 4aa07b7942086b3f5cf8d36026a50e8063312b10 Author: Jake Date: Mon Dec 21 01:38:01 2020 -0800 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80c2a41 --- /dev/null +++ b/.gitignore @@ -0,0 +1,248 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/pycharm,python +# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm,python + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# profiling data +.prof + +# End of https://www.toptal.com/developers/gitignore/api/pycharm,python \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..e76ba16 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/../../../../../../:\Users\jdc10\PycharmProjects\NikeBot\.idea/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/NikeBot.iml b/.idea/NikeBot.iml new file mode 100644 index 0000000..0c80114 --- /dev/null +++ b/.idea/NikeBot.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..18ebff9 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,98 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..cb3124b --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..c6996c1 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..47e56e6 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +NikeBot diff --git a/data/results b/data/results new file mode 100644 index 0000000..e69de29 diff --git a/src/Worker.py b/src/Worker.py new file mode 100644 index 0000000..a6469fe --- /dev/null +++ b/src/Worker.py @@ -0,0 +1,159 @@ +import logging + +from selenium import webdriver +from selenium.webdriver import Chrome, ChromeOptions + +# from msedge.selenium_tools.options import Options +# from msedge.selenium_tools import Edge +from selenium.common.exceptions import NoSuchElementException +import os +import pickle +import time +from colored import fg, attr +from uuid import uuid4 +from random import randrange, choice + +worker = None + + +class Worker(object): + def __init__( + self, username, password, cookie_name, port, headless=False, proxy=True + ): + self.username = username + self.password = password + self.cookie_name = cookie_name + self.headless = headless + self.driver = self.set_driver(headless, proxy, port) + self.uuid = uuid4() + self.fg = randrange(1, 7) + self.attr = choice([0, 1, 4]) + + def set_driver(self, headless, proxy, port) -> webdriver.Edge: + logging.info(port) + if os.path.isfile("webdrivers/chromedriver.exe"): + chrome_driver = "webdrivers/chromedriver.exe" + else: + print("No driver found.") + raise FileNotFoundError + options = ChromeOptions() + # options.add_argument('ignore-certificate-errors') + options.add_argument("disable-web-security") + # options.add_extension("data/webrtc.crx") + + if headless: + options.headless = True + + capabilities = webdriver.DesiredCapabilities.CHROME.copy() + if proxy: + capabilities["proxy"] = { + "httpProxy": f"192.168.1.52:{port}", + "ftpProxy": f"192.168.1.52:{port}", + "sslProxy": f"192.168.1.52:{port}", + "proxyType": "MANUAL", + } + else: + logging.warning("Not using proxy!") + + return Chrome( + options=options, + executable_path=chrome_driver, + desired_capabilities=capabilities, + ) + + def login(self): + driver = self.driver + driver.implicitly_wait(5) + driver.get("https://www.nike.com/login") + driver.find_element_by_css_selector("input[autocomplete='email']").send_keys(self.username) + driver.find_element_by_css_selector('input[autocomplete="current-password"]').send_keys(self.password) + driver.find_element_by_css_selector('input[value="SIGN IN"]').click() + + success = False + while not success: + time.sleep(2) + + try: + driver.implicitly_wait(1) + driver.find_element_by_css_selector('input[value="Dismiss this error"]').click() + driver.implicitly_wait(5) + driver.find_element_by_css_selector('input[autocomplete="current-password"]').send_keys(self.password) + driver.find_element_by_css_selector('input[value="SIGN IN"]').click() + except: + success = True + # if driver.find_element_by_css_selector() + + def go(self): + driver = self.driver + driver.implicitly_wait(2) + + logging.info(f"Starting on thing") + driver.get("http://lumtest.com/myip.json") + time.sleep(100) + + logging.error("Uh oh, I made it too far!") + + def close(self): + self.driver.close() + self.driver.quit() + + @staticmethod + def get_node_text(webElement): + text: str = webElement.text + for child in webElement.find_elements_by_xpath("./*"): + text = text.replace(child.text, "") + return text + + def print(self, string: str): + print( + f"{fg(self.fg)}{str(self.uuid)[:6]}--{string}{attr(self.attr)}", flush=True + ) + + def check_for_error(self, webElement): + pass + + +def initializer(login_list, port_list, log_queue, log_config, level=logging.DEBUG): + log_config(log_queue, level) + logging.info("init-start") + global worker + port = port_list.pop() + login_info = login_list.pop() + worker = Worker( + login_info["un"], + login_info["pw"], + login_info["name"], + port, + headless=False, + proxy=True, + ) + worker.login() + logging.info("init-fin") + + +def start(): + global worker + result = worker.go() + if result: + logging.info(f"Finished successfully! {result}") + else: + logging.warning(f"Failed! {result}") + return result + + +def before_exit(): + global worker + worker.close() + + +if __name__ == "__main__": + username = os.getenv("USERNAME") + password = os.getenv("PASSWORD") + + cdriver = Worker(username, password, headless=False) + cdriver.login() + + cdriver.go("Tyler", "Menezes", "Washington") + cdriver.go("Jacob", "Cover", "San Diego") + cdriver.go("TJ", "Horner", "San Diego") + cdriver.close() diff --git a/src/logger.py b/src/logger.py new file mode 100644 index 0000000..bb2a62a --- /dev/null +++ b/src/logger.py @@ -0,0 +1,58 @@ +# From https://docs.python.org/3.7/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes + +import logging +import logging.handlers +import multiprocessing + + +def listener_configurer(console_level): + root = logging.getLogger() + rfh = logging.handlers.RotatingFileHandler("../linkedout.log", "a", 1500000, 10) + rfh_f = logging.Formatter( + "%(asctime)s | %(processName)-10s | %(name)s | %(levelname)-8s | %(message)s" + ) + rfh.setFormatter(rfh_f) + root.addHandler(rfh) + sh = logging.StreamHandler() + sh.setLevel(console_level) + sh_f = logging.Formatter( + "%(processName)-10s %(levelname)-8s | %(message)s" + ) + sh.setFormatter(sh_f) + root.addHandler(sh) + root.info("Starting logging") + + +# This is the listener process top-level loop: wait for logging events +# (LogRecords)on the queue and handle them, quit when you get a None for a +# LogRecord. +def listener_process(queue, configurer, console_level=logging.INFO): + configurer(console_level) + while True: + try: + record = queue.get() + if ( + record is None + ): # We send this as a sentinel to tell the listener to quit. + break + logger = logging.getLogger(record.name) + logger.handle(record) # No level or filter logic applied - just do it! + except Exception: + import sys, traceback + + print("Whoops! Problem:", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + + +LEVELS = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL] + + +# The worker configuration is done at the start of the worker process run. +# Note that on Windows you can't rely on fork semantics, so each process +# will run the logging configuration code when it starts. +def worker_configurer(queue, level=logging.DEBUG): + h = logging.handlers.QueueHandler(queue) # Just the one handler needed + root = logging.getLogger() + root.addHandler(h) + # send all messages, for demo; no other level or filter logic applied. + root.setLevel(level) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..bc42b0a --- /dev/null +++ b/src/main.py @@ -0,0 +1,47 @@ +import os + +from src import Worker +from os import getenv +from multiprocessing import Pool, Queue, Process, Manager +import csv +import json +from src.logger import listener_process, listener_configurer, worker_configurer + + +def setup_logins_ports(): + manager = Manager() + shared_ip_list = manager.list([24000, 24000, 24000, 24001, 24001, 24001]) + login_list = manager.list([ + {"name": "johnpeter", "un": getenv("USERNAME1"), "pw": getenv("PASSWORD1")}, + {"name": "johnpeter", "un": getenv("USERNAME1"), "pw": getenv("PASSWORD1")}, + {"name": "johnpeter", "un": getenv("USERNAME1"), "pw": getenv("PASSWORD1")}, + {"name": "peterjohn", "un": getenv("USERNAME2"), "pw": getenv("PASSWORD2")}, + {"name": "peterjohn", "un": getenv("USERNAME2"), "pw": getenv("PASSWORD2")}, + {"name": "peterjohn", "un": getenv("USERNAME2"), "pw": getenv("PASSWORD2")}, + ]) + return shared_ip_list, login_list + + +if __name__ == "__main__": + num_processes = int(getenv("NUM_PROCESSES", 6)) + num_total_processes = int(getenv("NUM_PROCESSES", 1)) + + with open("data/results", "w") as results_file: + # Start the data at the last known chunk, so if something fails, only a little progress is lost + # Setup some logging things + log_queue = Queue(-1) + log_listener = Process(target=listener_process, args=(log_queue, listener_configurer)) + log_listener.start() + + # Loop through, one chunk at a time + for i in range(num_total_processes): + # Setup the list of ports and login information + shared_ip_list, login_list = setup_logins_ports() + with Pool(num_processes, initializer=Worker.initializer, + initargs=(login_list, shared_ip_list, log_queue, worker_configurer)) as p: + result = p.apply(Worker.start) + for i in range(num_processes): + p.apply(Worker.before_exit) + results_file.flush() + results_file.write(f"{json.dumps(result)} \n") + results_file.flush() diff --git a/webdrivers/chromedriver.exe b/webdrivers/chromedriver.exe new file mode 100644 index 0000000..14e61bc Binary files /dev/null and b/webdrivers/chromedriver.exe differ diff --git a/webdrivers/msedgedriver.exe b/webdrivers/msedgedriver.exe new file mode 100644 index 0000000..fcd5f75 Binary files /dev/null and b/webdrivers/msedgedriver.exe differ