فهرست منبع

it should be ok now.

lyc8503 3 سال پیش
والد
کامیت
1f86952f1c
4فایلهای تغییر یافته به همراه40 افزوده شده و 30 حذف شده
  1. 7 0
      backend_refactor/app.py
  2. 0 5
      backend_refactor/gunicorn.conf.py
  3. 31 23
      backend_refactor/pool/thread_pool.py
  4. 2 2
      backend_refactor/route/analyze.py

+ 7 - 0
backend_refactor/app.py

@@ -37,6 +37,13 @@ try:
 except:
     pass
 
+try:
+    for i in os.listdir(UPLOAD_DIR):
+        if i.endswith(".running"):
+            os.remove(UPLOAD_DIR + "/" + i)
+except:
+    pass
+
 
 @app.route("/")
 def hello_world():

+ 0 - 5
backend_refactor/gunicorn.conf.py

@@ -1,5 +1,4 @@
 # Sample Gunicorn configuration file.
-from pool.thread_pool import init_workers
 
 #
 # Server socket
@@ -183,10 +182,6 @@ proc_name = None
 #
 
 
-def on_starting(server):
-    server.log.info("Server is starting. Spawning pool threads.")
-    init_workers()
-
 
 def post_fork(server, worker):
     server.log.info("Worker spawned (pid: %s)", worker.pid)

+ 31 - 23
backend_refactor/pool/thread_pool.py

@@ -1,36 +1,44 @@
+import os
+import random
+import string
 import threading
-import queue
 import logging
-import functools
+import time
 
-q = queue.Queue()
+from config import UPLOAD_DIR
 
 
-def worker():
-    while True:
+def wait_and_execute(job, callback):
+    def worker():
+        while True:
+            count = 0
+            for i in os.listdir(UPLOAD_DIR):
+                if i.endswith(".running"):
+                    count += 1
+            if count < 2:
+                break
+            time.sleep(random.randint(1, 2))
+
+        name = ''.join(random.sample(string.ascii_letters + string.digits, 8))
+        with open(UPLOAD_DIR + "/" + name + ".running", "w") as f:
+            pass
+
+        logging.info("start job: " + name)
+
         try:
-            task = q.get()
-            logging.debug(f"Working on {task}")
-            job, callback = task
             job(callback)
-            logging.debug(f"Working on {task} done.")
-            q.task_done()
         except Exception as e:
-            logging.debug("error:", exc_info=e)
+            logging.debug("Worker error: ", exc_info=e)
             raise e
 
+        try:
+            os.remove(UPLOAD_DIR + "/" + name + ".running")
+            logging.info("finish job: " + name)
+        except:
+            pass
 
-def init_workers(count=2):
-    logging.info("Initializing workers...")
-    global q
-    q = queue.Queue()
-    for _ in range(count):
-        threading.Thread(target=worker).start()
-
-
-class Pool:
-    def submit(self, job, callback):
-        q.put((job, callback))
+    return worker
 
 
-pool = Pool()
+def submit(job, callback):
+    threading.Thread(target=wait_and_execute(job, callback)).start()

+ 2 - 2
backend_refactor/route/analyze.py

@@ -6,7 +6,7 @@ from flask import request
 
 from db import db
 from db.task import Task
-from pool.thread_pool import pool
+from pool.thread_pool import submit
 from route.util import make_response
 from service.photo_analyzer import StandingPhotoAnalyzer
 from service.video_analyzer import *
@@ -81,7 +81,7 @@ def analysis():
 
     logging.debug("task created.")
 
-    pool.submit(ana.analyze, callback_generator(task_uuid))
+    submit(ana.analyze, callback_generator(task_uuid))
 
     logging.debug("task submited")