fk filelock

backend_refactor/gunicorn.conf.py

@@ -65,7 +65,7 @@ backlog = 2048
 #       A positive integer. Generally set in the 1-5 seconds range.
 #
 
-workers = 5
+workers = 2
 worker_class = 'sync'
 worker_connections = 1000
 timeout = 15

backend_refactor/pool/thread_pool.py

@@ -1,44 +1,29 @@
-import os
-import random
-import string
-import threading
 import logging
 import time
+from threading import Thread
 
-from config import UPLOAD_DIR
-
-
-def wait_and_execute(job, callback):
-    def worker():
-        while True:
-            count = 0
-            for i in os.listdir(UPLOAD_DIR):
-                if i.endswith(".running"):
-                    count += 1
-            if count < 2:
-                break
-            time.sleep(random.randint(1, 2))
-
-        name = ''.join(random.sample(string.ascii_letters + string.digits, 8))
-        with open(UPLOAD_DIR + "/" + name + ".running", "w") as f:
-            pass
-
-        logging.info("start job: " + name)
 
+def process():
+    while True:
         try:
-            job(callback)
+            if len(queue) > 0:
+                fn = queue.pop()
+                fn()
         except Exception as e:
-            logging.debug("Worker error: ", exc_info=e)
-            raise e
+            logging.error(e)
+        finally:
+            time.sleep(1)
 
-        try:
-            os.remove(UPLOAD_DIR + "/" + name + ".running")
-            logging.info("finish job: " + name)
-        except:
-            pass
 
-    return worker
+# 使用 gunicorn 时, gunicorn 会 fork 出若干个子进程, 每个进程都有自己的 worker_thread, 且内存不共享
+worker_thread = None
+queue = []
 
 
 def submit(job, callback):
-    threading.Thread(target=wait_and_execute(job, callback)).start()
+    global worker_thread
+    if worker_thread is None:
+        worker_thread = Thread(target=process)
+        worker_thread.start()
+
+    queue.append(lambda: job(callback))