1.7.8 add new version check

feapder/VERSION

@@ -1 +1 @@
-1.7.8-beta2
+1.7.8

feapder/commands/cmdline.py

@@ -8,9 +8,12 @@
 @email: [email protected]
 """
 
+import re
 import sys
 from os.path import dirname, join
 
+import requests
+
 from feapder.commands import create_builder
 from feapder.commands import shell
 from feapder.commands import zip
@@ -32,12 +35,18 @@
 Available commands:
 """
 
+NEW_VERSION_TIP = """
+──────────────────────────────────────────────────────
+New version available \033[31m{version}\033[0m → \033[32m{new_version}\033[0m
+Run \033[33mpip install --upgrade feapder\033[0m to update!
+"""
 
-def _print_commands():
-    with open(join(dirname(dirname(__file__)), "VERSION"), "rb") as f:
-        version = f.read().decode("ascii").strip()
+with open(join(dirname(dirname(__file__)), "VERSION"), "rb") as f:
+    VERSION = f.read().decode("ascii").strip()
 
-    print(HELP.rstrip().format(version=version))
+
+def _print_commands():
+    print(HELP.rstrip().format(version=VERSION))
     cmds = {
         "create": "create project、spider、item and so on",
         "shell": "debug response",
@@ -49,6 +58,21 @@ def _print_commands():
     print('\nUse "feapder <command> -h" to see more info about a command')
 
 
+def check_new_version():
+    try:
+        url = "https://pypi.org/simple/feapder/"
+        resp = requests.get(url, timeout=3)
+        html = resp.text
+
+        last_version = re.findall(r"feapder-([\d.]*?).tar.gz", html)[-1]
+        now_stable_version = re.sub("-beta.*", "", VERSION)
+
+        if now_stable_version < last_version:
+            return f"feapder=={last_version}"
+    except:
+        pass
+
+
 def execute():
     try:
         args = sys.argv
@@ -68,6 +92,12 @@ def execute():
     except KeyboardInterrupt:
         pass
 
+    new_version = check_new_version()
+    if new_version:
+        version = f"feapder=={VERSION.replace('-beta', 'b')}"
+        tip = NEW_VERSION_TIP.format(version=version, new_version=new_version)
+        print(tip)
+
 
 if __name__ == "__main__":
     execute()

tests/test_template/test_spider.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2022-08-03 17:35:15
+Created on 2022-08-04 17:58:45
 ---------
 @summary:
 ---------
@@ -11,7 +11,7 @@
 from feapder import ArgumentParser
 
 
-class TestSpider(feapder.BatchSpider):
+class TestSpider(feapder.TaskSpider):
     # 自定义数据库，若项目中有setting.py文件，此自定义可删除
     __custom_setting__ = dict(
         REDISDB_IP_PORTS="localhost:6379",
@@ -25,7 +25,9 @@ class TestSpider(feapder.BatchSpider):
     )
 
     def start_requests(self, task):
-        yield feapder.Request("https://spidertools.cn")
+        task_id = task.id
+        url = task.url
+        yield feapder.Request(url, task_id=task_id)
 
     def parse(self, request, response):
         # 提取网站title
@@ -34,18 +36,26 @@ def parse(self, request, response):
         print(response.xpath("//meta[@name='description']/@content").extract_first())
         print("网站地址: ", response.url)
 
+        # mysql 需要更新任务状态为做完 即 state=1
+        yield self.update_task_batch(request.task_id)
+
 
 if __name__ == "__main__":
+    # 用mysql做任务表，需要先建好任务任务表
     spider = TestSpider(
-        redis_key="xxx:xxxx",  # 分布式爬虫调度信息存储位置
+        redis_key="xxx:xxx",  # 分布式爬虫调度信息存储位置
         task_table="",  # mysql中的任务表
-        task_keys=["id", "xxx"],  # 需要获取任务表里的字段名，可添加多个
+        task_keys=["id", "url"],  # 需要获取任务表里的字段名，可添加多个
         task_state="state",  # mysql中任务状态字段
-        batch_record_table="xxx_batch_record",  # mysql中的批次记录表
-        batch_name="xxx(周全)",  # 批次名字
-        batch_interval=7,  # 批次周期 天为单位 若为小时 可写 1 / 24
     )
 
+    # 用redis做任务表
+    # spider = TestSpider(
+    #     redis_key="xxx:xxxx",  # 分布式爬虫调度信息存储位置
+    #     task_table="", # 任务表名
+    #     task_table_type="redis", # 任务表类型为redis
+    # )
+
     parser = ArgumentParser(description="TestSpider爬虫")
 
     parser.add_argument(
@@ -66,4 +76,4 @@ def parse(self, request, response):
 
     # 通过命令行启动
     # python test_spider.py --start_master  # 添加任务
-    # python test_spider.py --start_worker  # 启动爬虫
+    # python test_spider.py --start_worker  # 启动爬虫