直播推流管理系统prtmp的开发过程总结 前言去年通过docker部署rtmp服务并进行网络压力测试，今年我们的业务终于用到了直播流服务。自己写了一个小系统用来从上游拉取直播流并分发到阿里云或者腾讯的直播服务。然后供小程序调用。特此记录整个配置过程。{card-default label="直播流管理" width="75%"}{/card-default}直播服务{message type="success" content="一般需要准备2个子域名，一个用来拉流，一个用来推流。可以多准备一个子域名，给自己写的流管理系统使用。"/}阿里云{callout color="#f0ad4e"}刚开始，用户不多，流量也很小，可以使用按量计费模式。{/callout}{card-default label="直播" width="75%"}{/card-default}自建直播服务{message type="success" content="直接用docker启动"/}参考: centos7部署rtmp服务并进行压力测试直播流管理系统{message type="success" content="初始功能比较简单，用到了flask+js+celery，用docker方式启动。"/}{card-describe title="主要文件"}main.py 视图，接口等函数utils.py 辅助函数{/card-describe}主要接口{message type="success" content="查询接口，推流接口,推流播放链接生成"/}推流url生成def ali_push_url(appName, streamName): """ 阿里云直播服务推流url """ #推流 push_domain = 'push.mytest.com' push_key = 'ZtBxxxxxxxMEKW' #过期时间 expire_time = 86400 time_stamp = int(time.time()) + expire_time #推流url pstr = '/{}/{}-{}-0-0-{}'.format(appName, streamName, time_stamp, push_key) pmd5 = md5_sign(pstr) purl= 'rtmp://{}/{}/{}?auth_key={}-0-0-{}'.format(push_domain, appName, streamName, time_stamp, pmd5) return purl直播服务urldef ali_live_url(appName, streamName): """ 阿里云直播服务播放url """ resp = {} #播放 play_domain = 'live.mytest.com' play_key = 'ulwxxxxxxxxxOm' #过期时间 expire_time = 86400 time_stamp = int(time.time()) + expire_time #播流url rstr = '/{}/{}-{}-0-0-{}'.format(appName, streamName, time_stamp, play_key) fstr = '/{}/{}.flv-{}-0-0-{}'.format(appName, streamName, time_stamp, play_key) hstr = '/{}/{}.m3u8-{}-0-0-{}'.format(appName, streamName, time_stamp, play_key) rmd5 = md5_sign(rstr) fmd5 = md5_sign(fstr) hmd5 = md5_sign(hstr) resp['rtmp_url'] = 'rtmp://{}/{}/{}?auth_key={}-0-0-{}'.format(play_domain, appName, streamName, time_stamp, rmd5) resp['flv_url'] = 'http://{}/{}/{}.flv?auth_key={}-0-0-{}'.format(play_domain, appName, streamName, time_stamp, fmd5) resp['hls_url'] = 'http://{}/{}/{}.m3u8?auth_key={}-0-0-{}'.format(play_domain, appName, streamName, time_stamp, hmd5) return resp推流任务函数@celery.task def push_rtmp_task(sid, source, target): """ 推流任务函数 """ print(source) print(target) # 视频源输入参数 input_args = ['-i', source] # 推流输出参数 output_args = ['-vcodec', 'libx264', '-acodec', 'aac', '-f', 'flv', target] # 合并参数 command = ['ffmpeg'] + input_args + output_args # 执行命令 with subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) as process: process.wait() # 更新推流信息 living_info = load_json('./data/living.json') if sid in living_info.keys(): living_info.pop(sid) save_living_info(living_info)前端功能{message type="success" content="通过js做数据交互"/}手动推流 //手动推流 function ManPushRtmp(dom) { var mymessage = confirm("确认手动推送此直播流到" + $(dom).attr("ptype") + "？" + $(dom).attr("url")); if (mymessage == true) { $('#myModal').modal('show'); $.ajax({ url : '/api/rtmppush?url=' + $(dom).attr("url") + '&utype=' + $(dom).attr("utype") + '&sid=' + $(dom).attr("sid") + '&mid=' + $(dom).attr("mid") + '&lid=' + $(dom).attr("lid") + '&ptype=' + $(dom).attr("ptype"), type : 'get', success : function(data) { $('#myModal').modal('hide'); if (data.code == 1000){ $("#child_table").bootstrapTable('refresh', data.data); alert(data.msg); } else { alert("推流失败! " + data.msg) } }, error : function(data){ $('#myModal').modal('hide'); alert("接口异常! " + data.msg) } }); } };停止推流 //停止推流 function ManStopPush(dom) { var mymessage = confirm("确认手动停止此直播流？" + $(dom).attr("stream_id")); if (mymessage == true) { $('#myModal').modal('show'); $.ajax({ url : '/api/stoppush?stream_id=' + $(dom).attr("stream_id"), type : 'get', success : function(data) { $('#myModal').modal('hide'); if (data.code == 1000){ alert(data.msg); location.reload(); } else { alert("失败! " + data.msg) } }, error : function(data){ $('#myModal').modal('hide'); alert("接口异常!" + data.msg) } }); } };异步任务{message type="success" content="通过celery实现异步任务执行"/}celery实例化from celery import Celery # celery配置 app.config['CELERY_BROKER_URL'] = 'redis://redis:6379/0' app.config['CELERY_RESULT_BACKEND'] = 'redis://redis:6379/0' #celery实例化 celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL']) celery.conf.update(app.config)caddy代理prmtp.mytest.com { tls admin@mytest.com encode gzip log { output file /opt/logs/access.log } header / { Strict-Transport-Security "max-age=31536000;includeSubdomains;preload" } #访问认证 #密码:123456 basicauth /* { admin $2a$14$DIjtbTxbUSZHfHJUrjuU9.45SlrcwICIXNVSwVxehsnHhTXBBN Nsi } ## HTTP 代理配置, 后端服务端口 reverse_proxy http://backend:5000 }启动#手动启动 python main.py celery -A main.celery worker -l info # 项目启动 docker-compose up -dFAQjs传递url参数需要转码{message type="success" content="使用encodeURIComponent函数"/}

通过Django管理周期性任务-feapder爬虫 前言{callout color="#f0ad4e"}一开始感兴趣的信息比较少,直接用crontab启动就满足要求了。后台爬虫越来越多，有的爬虫早就失效了，也没发现。用了 feapder 作者的管理系统 feaplat 。系统功能很全面，但是随着功能的完善，价格也越来越贵。个人实在承担不起，只能花时间自己搞一个简易版的了。{/callout}{message type="success" content="我自己感兴趣的信息收集回来通过Django管理，把爬虫功能顺便集成到里面了。"/}{card-default label="项目" width="75%"}{/card-default}功能实现模型设计隐藏内容，请前往内页查看详情后台管理{message type="success" content="默认就能用,通过admin.py可以自定义需要显示的内容。"/}# Register your models here. class SpiderInfofAdmin(admin.ModelAdmin): #后台展示字段 list_display = ['id', 'sname', 'filepath', 'workpath', 'image', 'addtime', 'snote'] #搜索字段 search_fields = ['sname'] class SpiderTaskAdmin(admin.ModelAdmin): #后台展示字段 list_display = ['id', 'sname', 'snote', 'addtime', 'runtime_show', 'total', 'repeat', 'valid', 'status_colored', 'operate'] #过滤字段 list_filter = ["status"] #搜索字段 search_fields = ['sname'] #只读字段 readonly_fields = ['id', 'addtime', 'runtime', 'total', 'repeat', 'valid', 'status'] #自定义动作 actions = ['schedule_switch']{card-default label="任务界面" width="75%"}{/card-default}异步任务{message type="success" content="异步任务和周期性任务通过celery实现"/}项目主settings.py添加内容INSTALLED_APPS = [ # 略 'django_celery_beat', #略 ] # Celery配置 # BROKER和BACKEND配置，这里用了本地的redis，其中1和2表示分别用redis的第一个和第二个db CELERY_BROKER_URL = 'redis://172.17.0.10:6379/1' CELERY_RESULT_BACKEND = 'redis://172.17.0.10:6379/2' # CELERY 时间 CELERY_TIMEZONE = TIME_ZONE DJANGO_CELERY_BEAT_TZ_AWARE = False #指定任务接收的内容序列化类型 CELERY_ACCEPT_CONTENT = ['application/json'] #任务和任务结果序列化方式 CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' #超过时间 CELERY_TASK_RESULT_EXPIRES = 12 * 30 #是否压缩 CELERY_MESSAGE_COMPRESSION = 'zlib' #并发数默 CELERYD_CONCURRENCY = 2 #celery worker 每次去redis取任务的数量认已CPU数量定 CELERYD_PREFETCH_MULTIPLIER = 2 #每个worker最多执行3个任务就摧毁，避免内存泄漏 CELERYD_MAX_TASKS_PER_CHILD = 3 #可以防止死锁 CELERYD_FORCE_EXECV = True #celery 关闭UTC时区 CELERY_ENABLE_UTC = False #celery 并发数设置，最多可以有20个任务同时运行 CELERYD_CONCURRENCY = 20 CELERYD_MAX_TASKS_PER_CHILD = 4 #celery开启数据库调度器，数据库修改后即时生效 CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler' #解决告警 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'2.同目录下新增celery.pyimport os from celery import Celery,platforms from django.conf import settings # 设置环境变量 os.environ.setdefault('DJANGO_SETTINGS_MODULE','taskmon.settings') # 实例化 app = Celery('taskmon') # namespace='CELERY'作用是允许你在Django配置文件中对Celery进行配置 # 但所有Celery配置项必须以CELERY开头，防止冲突 app.config_from_object('django.conf:settings', namespace='CELERY') # 自动从Django的已注册app中发现任务 app.autodiscover_tasks() #允许root 用户运行celery platforms.C_FORCE_ROOT = True # 一个测试任务 @app.task(bind=True) def debug_task(self): print('Request: {0!r}'.format(self.request))3.项目目录下tasks.py#操作docker from celery import shared_task from .utils import process_start @shared_task def sync_start_process(sname): """ 异步执行任务 """ process_start(sname)4.celery启动#任务调度 celery multi start worker -A taskmon -l info --logfile=/logs/celery_worker.log celery -A taskmon beat -l info --logfile=/logs/celery_beat.log{card-default label="运行日志" width="75%"}{/card-default}添加周期任务隐藏内容，请前往内页查看详情删除周期任务def remove_celery_task(sid): """ 删除计划任务 sid : 爬虫任务ID """ cname = str(sid) + '-' + '周期任务' #添加计划任务 with transaction.atomic(): save_id = transaction.savepoint() try: _p = PeriodicTask.objects.get(name=cname) if _p: _p.delete() print('{}删除计划任务成功'.format(cname)) return True except Exception as e: transaction.savepoint_rollback(save_id) print('{}删除计划任务失败，错误原因：'.format(cname) + str(e)) return False任务启动函数def process_start(sname): """ 执行任务并处理返回结果 sname: 任务名 cinfo: 启动容器所需的信息 """ con_name = 'spider_{}_1'.format(sname) containers = get_containers({"name":con_name}) if containers: print('有相同任务运行中...|{}|{}'.format(con_name, datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))) return False #查询库 spider_task = SpiderTask.objects.get(sname=sname) #构建docker启动信息 cinfo = { "name": con_name, "command": spider_task.command, #宿主机目录 "volumes": ['/opt/project/taskmon/myapp/spider/{}:{}'.format(spider_task.sinfo.filepath, spider_task.sinfo.workpath),], "shm_size": spider_task.sinfo.shm, "image": spider_task.sinfo.image, "working_dir": spider_task.sinfo.workpath, "remove": False, } #启动容器 result = run_container(cinfo) if result: #日志文件 log_path='/logs/{}_{}.log'.format(con_name, datetime.datetime.now().strftime("%H-%M-%S-%Y-%m-%d")) #保存日志 with open(log_path, 'wb') as fw: fw.write(result) #采集结果 d_nums = process_result(result) #更新 spider_task.total = d_nums[0] spider_task.repeat = d_nums[1] spider_task.valid = d_nums[2] spider_task.logpath = log_path spider_task.save() print('任务执行...|{}|{}'.format(con_name, datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")))容器启动函数def run_container(ddict): """ 运行容器 """ #print(ddict) container = client.containers.run( image=ddict['image'], name=ddict['name'], shm_size=ddict['shm_size'], volumes=ddict['volumes'], working_dir=ddict['working_dir'], remove=ddict['remove'], detach=True, command=ddict['command'] ) container.wait() result = container.logs() container.remove() return result结果处理函数{message type="success" content="统计每次处理收到的结果"/}def process_result(result): """ 处理返回结果 """ a = 0 b = 0 c = 0 lines = str(result, encoding = "utf-8").split('\n') for line in lines: if '待入库数据' in line: tmp_s = line.split('|')[3] nums = tmp_s.split(' ') a += int(nums[2]) b += int(nums[5]) c += int(nums[7]) return (a, b, c){card-default label="效果" width="80%"}{/card-default}相关文章Django后台展示（一） Django后台展示（二）FAQ自动添加周期任务后，启动报错{card-default label="报错" width="85%"}{/card-default}{message type="success" content="解决办法: 写入数据库django_celery_beat_periodictask表的args字段需要双引号括起来。"/}

爬虫管理系统FEAPLAT部署与使用 前言去年用feapder写了几个小爬虫，定时收集自己感兴趣的数据，运行的挺稳定的。最近又添加了几个新的，同时发现原先写的有几个早就失效了，一直没有发现。于是花时间折腾一下官方的 feaplat爬虫管理系统，新版本好像可以免费20个采集任务,够自己用了。部署安装dockerdocker和docker-compose一键安装脚本# 配置 docker swarm docker swarm init ## 有多块网卡的话 docker swarm init --advertise-addr 172.17.0.10docker-compose.yaml{message type="success" content="我的服务器上已经安装了mysql和redis，改造一下docker-compose.yaml，直接使用自己的了。"/}version: "3.5" x-logging: &default-logging options: max-size: "100m" max-file: "1" driver: json-file services: feapder_front: container_name: feapder_front image: ${FRONT_IMAGE} restart: always environment: - BACKEND_URL=http://feapder_backend:${BACKEND_PORT} - FRONT_PORT=${FRONT_PORT} ports: - ${FRONT_PORT}:${FRONT_PORT} # 前端端口 （自定义端口:80） depends_on: - feapder_backend logging: *default-logging feapder_backend: container_name: feapder_backend image: ${BACKEND_IMAGE} restart: always command: /wait-for-it.sh 172.17.0.10:3306 -t 60 --strict -- uvicorn main:app --host 0.0.0.0 --workers ${BACKEND_WORKER} --port ${BACKEND_PORT} # workers 为后端服务的个数，爬虫多可改大点 ports: - ${BACKEND_PORT}:${BACKEND_PORT} # 后端端口 （自定义端口:8000） environment: - FEAPDER_BACKEND_URL=http://${BACKEND_HOST}:${BACKEND_PORT} - AUTHORIZATION_CODE=${AUTHORIZATION_CODE} # 授权码 - DB_URL=mysql+pymysql://fpuser:fpuserxxx@172.17.0.10:3306/feapder_platform?charset=utf8mb4 # 后端数据库配置 - REDIS_DB_URL=redis://:@172.17.0.10:6379/0 # redis数据库连接配置 redis://[[username]:[password]]@[host]:[port]/[db] - ACCESS_TOKEN_EXPIRE_MINUTES=1440 # 管理系统账号cookie过期时间 单位分钟 - SPIDER_IMAGE=${SPIDER_IMAGE} # 爬虫镜像 - SPIDER_AUTO_PULL_IMAGE=1 # 是否自动拉取镜像 否则需要在爬虫节点手动 docker pull 爬虫镜像，为了加快启动速度，可以设置0 - SPIDER_ENV={} # 爬虫环境变量 值为json类型 # 爬虫容器启动参数，支持的参数使用 docker service create --help 查看 - SPIDER_RUN_ARGS={"--network":"feaplat"} # git ssh 私有密钥，不填则使用默认的 - GIT_SSH_PRIVATE_KEY=${GIT_SSH_PRIVATE_KEY} # 监控配置 - INFLUXDB_HOST=${INFLUXDB_HOST} - INFLUXDB_DB=${INFLUXDB_DB} - INFLUXDB_ADMIN_USER=${INFLUXDB_ADMIN_USER} - INFLUXDB_ADMIN_PASSWORD=${INFLUXDB_ADMIN_PASSWORD} - INFLUXDB_PORT_TCP=8086 - INFLUXDB_PORT_UDP=8089 volumes: - "/var/run/docker.sock:/var/run/docker.sock" - "~/data/feapder/projects:/projects" # 上传的项目挂载， 本地目录:容器路径 depends_on: - influxdb logging: *default-logging feapder: container_name: feapder_worker image: ${SPIDER_IMAGE} influxdb: restart: always image: registry.cn-hangzhou.aliyuncs.com/feapderd/influxdb:1.8.6 container_name: feapder_influxdb volumes: - ~/data/feapder/influxdb:/var/lib/influxdb - ./conf/influxdb.conf:/etc/influxdb/influxdb.conf environment: - INFLUXDB_DB=${INFLUXDB_DB} - INFLUXDB_ADMIN_USER=${INFLUXDB_ADMIN_USER} - INFLUXDB_ADMIN_PASSWORD=${INFLUXDB_ADMIN_PASSWORD} - INFLUXDB_USER=influx - INFLUXDB_USER_PASSWORD=influx ports: - ${INFLUXDB_PORT_TCP}:8086 - ${INFLUXDB_PORT_UDP}:8089/udp logging: *default-logging networks: default: name: feaplat driver: overlay attachable: true.env{message type="error" content="可以自定义端口和镜象版本爬虫版本等"/}启动docker-compose up -d访问http://ip:804 {callout color="#f0ad4e"}默认账号密码:admin/admin{/callout}{card-default label="登录页面" width="75%"}{/card-default}其他数据表手动查询{message type="success" content="表结构挺简单的"/}# 查询所有任务 select * from feapder_task\G ## 修改任务自增id起始点，清理过期任务后 ALTER TABLE feapder_task AUTO_INCREMENT = 1; ## 修改任务id update feapder_task set id = 2 where id =23;{card-default label="任务信息" width="80%"}{/card-default}手动启动爬虫#启动 docker service create --name task_test --replicas 1 --workdir /task --mount type=bind,src=/opt/project/taskmonitor/myapp/spider/task,dst=/task registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.3 tailf test.py #查看 docker service ps pzsutx0dhpkmngrs7roarsa6r #删除 docker service rm pzsutx0dhpkm

利用aria2分析并下载torrent种子 前言{callout color="#f0ad4e"}上一篇文章分享了如何将吃灰的旧手机改造成服务器，大功告成。就要利用起来，平时就用来下剧吧。{/callout}安装apt inatall -y aria2使用下载直链# 直链 aria2c http://xx.com/xx # 重命名yy aria2c -o yy http://xx.com/xx # 分2线程2段下载 aria2c -s 2 -x 2 http://xx.com/xxbt下载# 下载整个种子 aria2c a.torrent # 磁力链接 aria2c '链接' # 列出种子内容 aria2c -S a.torrent # 下载种子内198，226,227,228的文件到指定目录 screen aria2c -d /data/res/th --select-file=198,226-228 a.torrent限速# 单文件 aria2c --max-download-limit=300K -s10 -x10 'http://xx.com/xx' # 整体限速 aria2c --max-overall-download-limit=300k -s10 -x10 'http://xx.com/xx' 批量修改文件名#导入模块 import os import re # 保留中文、大小写、数字 def remove_special_characters(old_s): # 匹配不是中文、大小写、数字的其他字符 cop = re.compile("[^\u4e00-\u9fa5^a-z^A-Z^0-9]") # 将old_s中匹配到的字符替换成空s字符 nwe_s = cop.sub('_', old_s) return nwe_s #得到当前目录下所有的文件 def get_all_dir(path, sp = ""): filesList = os.listdir(path) #处理每一个文件 sp += "-" for filename in filesList: #判断一个文件是否为目录(用绝对路径) join拼判断接法 if os.path.isdir(os.path.join(path, filename)): final_filename = remove_special_characters(filename) print("{} {}||{}".format(sp, filename, final_filename)) #重命名 os.rename(os.path.join(path, filename), os.path.join(path, final_filename)) #递归调用 自己调用自己 get_all_dir(os.path.join(path, final_filename), sp) else: temp_filename, file_extension = os.path.splitext(filename) final_filename = remove_special_characters(temp_filename) + file_extension print("{} {}||{}".format(sp, filename, final_filename)) #重命名 os.rename(os.path.join(path, filename), os.path.join(path, final_filename)) #等待用户确认函数 def wait_commit(): ''' 请用户手动输入yes确认操作 ''' tag = input('请手动输入yes确认操作:') if tag == 'yes' or tag == 'y': return True else: return False #主函数 if __name__ == '__main__': #待处理的目录 fdir=os.getcwd() print('将要批量修改文件名! 目录:{}'.format(fdir)) if wait_commit(): get_all_dir(fdir) else: print('被拒绝,退出')文件服务器caddy file-server --browse --root ~/Downloads/thFAQaria2c下载报错{callout color="#f0ad4e"}Exception caught while saving DHT routing table to /root/.cache/aria2/dht.dat{/callout}mkdir /root/.cache/aria2/ && cd /root/.cache/aria2/ wget https://github.com/P3TERX/aria2.conf/blob/master/dht.dat

基于docker的TinyTinyRSS部署过程 前言{callout color="#f0ad4e"}RSS 就是信息聚合服务，把你想看的内容通过 RSS 订阅，就可以起到筛选的作用。避免平台系统推荐引发的信息茧房效应。正好手里有可用的vps，顺便部署了一个，特此记录部署过程。{/callout}准备{callout color="#f0ad4e"}域名一个,并做好解析：n.webzhan.xyzvps一台，推荐 香港节点 ，省去备案的过程。{/callout}系统初始化{message type="success" content="系统初始化，并安装docker和docker-compose"/}参考文章： docker和docker-compose一键安装脚本启动ttrss创建目录mkdir ttrss编辑docker-compose.yamlversion: "3" networks: tnet: driver: bridge services: database.postgres: image: postgres:13-alpine container_name: postgres environment: - POSTGRES_PASSWORD=postxxx@123 # please change the password volumes: - ~/postgres/data/:/var/lib/postgresql/data # persist postgres data to ~/postgres/data/ on the host restart: always networks: - tnet service.rss: image: wangqiru/ttrss:latest container_name: ttrss ports: - 181:80 environment: - SELF_URL_PATH=https://n.webzhan.xyz/ # please change to your own domain - DB_HOST=database.postgres - DB_PORT=5432 - DB_NAME=ttrss - DB_USER=postgres - DB_PASS=postxxx@123 # please change the password - ENABLE_PLUGINS=auth_internal,fever # auth_internal is required. Plugins enabled here will be enabled for all users as system plugins - FEED_LOG_QUIET=true stdin_open: true tty: true restart: always command: sh -c 'sh /wait-for.sh $$DB_HOST:$$DB_PORT -- php /configure-db.php && exec s6-svscan /etc/s6/' networks: - tnet service.mercury: # set Mercury Parser API endpoint to `service.mercury:3000` on TTRSS plugin setting page image: wangqiru/mercury-parser-api:latest container_name: mercury expose: - 3000 restart: always networks: - tnet service.opencc: # set OpenCC API endpoint to `service.opencc:3000` on TTRSS plugin setting page image: wangqiru/opencc-api-server:latest container_name: opencc environment: - NODE_ENV=production expose: - 3000 restart: always networks: - tnet启动项目cd ttrss docker-compose up -d利用caddy代理参考： 利用Caddy替代nginx做代理服务器 {card-describe title="配置信息"}n.webzhan.xyz { tls admin@webzhan.xyz encode gzip log { output file /opt/logs/access.log } header / { Strict-Transport-Security "max-age=31536000;includeSubdomains;preload" } ## HTTP 代理配置, ttrss服务IP地址+端口 reverse_proxy 127.0.0.1:181 }{/card-describe}访问https://n.webzhan.xyz/ {callout color="#f0ad4e"}账号: admin密码: password{/callout}{card-default label="文章" width="90%"}{/card-default}{message type="error" content="最后添加自己喜欢的rss源即可。"/}设置备份{card-default label="备份" width="75%"}{/card-default}添加源关注知乎问题https://rss.lilydjwg.me/zhihu_question/40854395知乎专栏https://rss.lilydjwg.me/zhihuzhuanlan/c_1433942042578157568FAQ重置密码UPDATE ttrss_users SET pwd_hash = 'SHA1:5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8' WHERE id = 1;

兰空图床开源版安装部署记录 前言{callout color="#f0ad4e"}一直想部署一个自己的图床应用，没有找到合适的源码。最近发现兰空图床，作者一直在更新。部署测试一下，特此记录搭建过程。{/callout}获取源码项目地址安装过程基本环境{message type="warning" content="PHP要求8.0.2版本以上，原先打包好的基于docker的lnmp包需要更新。改动文件如下："/}{card-describe title="Dockerfile"}FROM php:8.0-fpm RUN usermod -u 1010 www-data COPY sources.list /etc/apt/sources.list RUN apt-get update \ # 相关依赖必须手动安装 && apt-get install -y \ libfreetype6-dev \ libjpeg62-turbo-dev \ libmcrypt-dev \ libpng-dev \ libzip-dev \ imagemagick\ libmagick++-dev\ # 安装扩展 && docker-php-ext-configure gd --with-freetype --with-jpeg \ && docker-php-ext-install -j$(nproc) gd iconv \ && docker-php-ext-install -j$(nproc) mysqli \ && docker-php-ext-install -j$(nproc) zip \ && docker-php-ext-install -j$(nproc) bcmath \ && docker-php-ext-install -j$(nproc) pdo pdo_mysql RUN pecl install imagick RUN docker-php-ext-enable imagick #COPY redis-5.3.7.tgz /tmp/redis-5.3.7.tgz #RUN tar xvf /tmp/redis-5.3.7.tgz -C /tmp/ #RUN mkdir -p /usr/src/php/ext/ && cp -r /tmp/redis-5.3.7 /usr/src/php/ext/redis #RUN docker-php-ext-install redis RUN apt-get purge -y g++ \ && apt-get autoremove -y \ && rm -r /var/lib/apt/lists/* \ && rm -rf /tmp/* #COPY composer.phar /usr/local/bin/composer #RUN chmod +x /usr/local/bin/composer RUN usermod -u 1010 www-data EXPOSE 9000 CMD ["php-fpm"]{/card-describe}安装过程{card-default label="环境检查" width="80%"}{/card-default}{card-default label="数据库配置" width="80%"}{/card-default}{card-default label="完成" width="80%"}{/card-default}访问{card-default label="首页" width="80%"}{/card-default}{card-default label="后台" width="95%"}{/card-default}完整项目文件获取【源码部署】兰空图床源码部署教程

测试用例管理系统TestKiss部署过程记录 前言{callout color="#f0ad4e"}本系统为独立的用例管理系统，测试的小伙伴需要管理测试用例。记录一下部署过程。{/callout}{card-default label="后台" width="95%"}{/card-default}项目地址TestKiss配置过程修改config.py配置 SERVER_URL = 'http://外网IP:8080' MONGO_URI = "mongodb"初次运行mongodb启动和初始化docker exec -it testkiss_python_1 bash python init_mongo.py mongodb基于docker部署{message type="success" content="采用docker-compose的部署方式"/}docker-compose.ymlversion: "3" networks: tnet: services: python: build: context: . dockerfile: ./docker/Dockerfile network: host volumes: - ./backend:/app - /var/log/testkiss/python/:/logs/ environment: - "SET_CONTAINER_TIMEZONE=true" - "TZ=Asia/Shanghai" working_dir: /app #command: python app.py command: tail -f app.py restart: unless-stopped depends_on: - mongodb ports: - "8080:8080" networks: - tnet mongodb: image: mongo:3.7 restart: always environment: - MONGO_DATA_DIR=/data/db - MONGO_LOG_DIR=/data/logs volumes: - /data/testkiss/mongodb:/data/db - /var/log/testkiss/mongodb:/data/logs ports: - "27017:27017" networks: - tnetDockerfileFROM python:3.6.8 RUN mkdir /pip ADD ./backend/requirements.txt /pip RUN pip install --upgrade pip RUN pip install -i https://mirrors.aliyun.com/pypi/simple/ -r /pip/requirements.txt启动docker-compose up -d完整项目代码获取完整代码+docker启动文件

推广管理系统开发过程总结 前言{callout color="#f0ad4e"}业务系统有一个分享返现的活动。用户在知乎或者小红书发表一篇使用心得，然后申请返现。已经支付的订单就会返还。原先订单少的时候，都是手动处理。随着订单量的增加，申请返现的也越来越多。有点处理不过来，同时已经返现的订单，有多少也不知道。于是有开发一个推广管理系统的想法，花一周时间实现了。特此记录实现过程，一切从简能用就好。{/callout}{card-default label="后台" width="80%"}{/card-default}设计思路{callout color="#f0ad4e"}后端采用自己比较熟悉的django框架，数据库使用mysql。前端页面使用html+js。几个页面比较简单。以后使用过程中逐步完善吧。{/callout}photo模块{message type="success" content="主要功能模块，记录推广链接和结算信息。"/}模型设计隐藏内容，请前往内页查看详情后台展示自定义{message type="success" content="方便后台管理，统计, 自定义展示。"/}隐藏内容，请前往内页查看详情新增订单接口{message type="success" content="提供给frontend模块使用的api接口"/}def photo_add(request): #定义返回字典 resp = {} #获取请求参数 if request.method == 'GET': u = request.GET.get('u', default=False) n = request.GET.get('n', default='xwzy1130') else: u = False if not u: resp['msg'] = '参数非法,提交失败!' else: data = get_info_by_url(u) if data: art = Article() art.sno = datetime.now().strftime('%Y%m%d%H%M%S') + ''.join(str(i) for i in random.sample(range(0,9),4)) art.title = data[0] art.url = u art.stype = data[1] payinfo = PayInfo.objects.filter(username=n).last() if payinfo: art.uid = payinfo.uid else: art.uid = 1 art.note = n art.save() resp['msg'] = '更新成功!查询码:{}'.format(art.sno) else: resp['msg'] = '提交失败!' return JsonResponse(resp)frontend模块{message type="success" content="前端展示页面，实现客户订单状态查询功能。"/}view模版#搜索结果页 def search_view(request,*args, **kwargs): return render(request, 'frontend/search.html') #自助提交页 def submit_view(request,*args, **kwargs): return render(request, 'frontend/submit.html')submit.html{message type="success" content="通过ajax提交数据"/} $(function () { // 搜索时执行ajax 查询匹配的数据 返回到list $(".handIpt").click(function () { var username = $("#username").val(); var url = $("#url").val(); if (isValidURL(url) && url.trim()) { $.ajax({ type: "GET", url: "/photo/add?u=" + url + "&n=" + username, dataType: "json", success: function (data) { alert(data.msg); }, //打印错误 error:function(jqXHR,textStatus,errorThrown){ console.log(jqXHR); console.log(textStatus); console.log(errorThrown); } }); } else { alert("请检查推广链接!"); } });{card-default label="提交页面" width="75%"}{/card-default}其他细节js校验url是否合法 function isValidURL(str_url) { var strRegex = "^((https|http|ftp|rtsp|mms)?://)" + "?(([0-9a-z_!~*'().&=+$%-]+: )?[0-9a-z_!~*'().&=+$%-]+@)?" //ftp的user@ + "(([0-9]{1,3}.){3}[0-9]{1,3}" // IP形式的URL- 199.194.52.184 + "|" // 允许IP和DOMAIN（域名） + "([0-9a-z_!~*'()-]+.)*" // 域名- www. + "([0-9a-z][0-9a-z-]{0,61})?[0-9a-z]." // 二级域名 + "[a-z]{2,6})" // first level domain- .com or .museum + "(:[0-9]{1,4})?" // 端口- :80 + "((/?)|" // a slash isn't required if there is no file name + "(/[0-9a-z_!~*'().;?:@&=+$,%#-]+)+/?)$"; var re=new RegExp(strRegex); //re.test() if (re.test(str_url)){ return (true); }else{ return (false); } }js 转码url // var e_url = encodeURIComponent (list[i].url); function movieDetail (url) { console.log(url); window.location.href= decodeURIComponent(url); }Django设置静态文件目录STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, 'collect_static') STATICFILES_DIRS = [ os.path.join(BASE_DIR, 'static') ]simpleUI 设置默认首页#simpleui配置 SIMPLEUI_HOME_PAGE = '/photo/index'

利用Django和feapder定制开发商品价格监控系统 前言{message type="success" content="最近想升级一下电脑硬件，又不是很了解当前电脑硬件的价格趋势。登录一些比价网站，商品太多了，找起来好麻烦。一些高级的功能还要收费。干脆自己写一个吧。想要的功能很简单："/}{callout color="#f0ad4e"}添加感兴趣的商品ID,自动获取商品信息添加是否获取价格和当前优惠活动开关，若开启，每天获取一次当前商品价格和活动将获取的商品价格可视化展示{/callout}{card-default label="数据管理" width="95%"}{/card-default}{card-describe title="主要技术"}Django:作为基础框架，用来管理商品信息和价格数据feapder:用来采集商品价格数据并写入数据库simpleui:Django后台ui美化pyecharts：数据可视化展示{/card-describe}Django 平台models.py{message type="success" content="数据模型设计"/}from django.db import models # Create your models here. #状态 NO = 0 YES = 1 STATUS_CHOICE = { NO: '否', YES: '是', } class MallInfo(models.Model): '''商品信息''' sid = models.AutoField(primary_key=True, verbose_name='编号') status_choices = ((k, v) for k,v in STATUS_CHOICE.items()) sku = models.CharField(max_length=32, verbose_name='商品id') name = models.CharField(max_length=255, verbose_name='商品名', default='xxxx') src = models.CharField(max_length=20, verbose_name='来源', default='jd') url = models.CharField(max_length=255, verbose_name='链接', default='xxxx') addtime = models.DateTimeField(auto_now_add=True, verbose_name='添加时间') status = models.SmallIntegerField(default=NO, choices=status_choices, verbose_name='是否监控') # admin显示订单的id def __str__(self): return self.name class Meta: db_table = 'mall_info' verbose_name = '商品信息' verbose_name_plural = '商品信息' class PriceInfo(models.Model): '''历史价格''' sid = models.AutoField(primary_key=True, verbose_name='编号') sku = models.CharField(max_length=32, verbose_name='商品id') price = models.IntegerField(default=0, verbose_name='价格') note = models.CharField(max_length=255, default='无', verbose_name='活动') addtime = models.DateTimeField(auto_now_add=True, verbose_name='日期') # admin显示订单的id def __str__(self): return str(self.sid) class Meta: db_table = 'price_info' verbose_name = '商品价格' verbose_name_plural = '商品价格'admin.py{message type="success" content="后台数据展示"/}from django.contrib import admin from mall.models import MallInfo, PriceInfo from django.utils.html import format_html from bs4 import BeautifulSoup import requests # Register your models here. class MallInfoAdmin(admin.ModelAdmin): list_display = ['sku', 'name_and_url', 'show_history', 'src', 'addtime', 'status'] #进入编辑的字段 list_display_links = ['sku'] #只读字段 readonly_fields = ['url'] #标题带链接 def name_and_url(self, obj): return format_html("<a href='{url}' target='_blank'>{name}</a>", url=obj.url, name=obj.name) #字段描述 name_and_url.short_description = "商品名称" def show_history(self, obj): return format_html("<a href='/mall/echarts/?sku={sku}' target='_blank'>历史价格</a>", sku=obj.sku) show_history.short_description = "历史价格" #重写保存函数 def save_model(self, request, obj, form, change): def get_name_by_sku(): """根据sku获取名称""" url = 'https://item.jd.com/%s.html' % (obj.sku) try: resp = requests.get(url) soup = BeautifulSoup(resp.text, 'html.parser') name = soup.find(class_='sku-name') return name.text.strip() except Exception as e: print(e) return '0000' if not change: # 新增记录 obj.name = get_name_by_sku() obj.url = 'https://item.jd.com/%s.html' % (obj.sku) super(MallInfoAdmin, self).save_model(request, obj, form, change) class PriceInfoAdmin(admin.ModelAdmin): list_display = ['sku', 'price', 'note', 'addtime'] #只读字段 readonly_fields = ['sku', 'price', 'note', 'addtime'] admin.site.register(MallInfo, MallInfoAdmin) #admin.site.register(PriceInfo, PriceInfoAdmin){card-default label="商品编辑" width="95%"}{/card-default}views.py{message type="success" content="api接口和页面视图函数"/}from django.shortcuts import render from mall.models import MallInfo, PriceInfo from django.http import HttpResponse from django.template import Template, Context from rest_framework.views import APIView import json from pyecharts.charts import Bar from pyecharts import options as opts from .utils import * # Create your views here. JsonResponse = json_response JsonError = json_error def get_skus_need_monitor(request): """ 获取需要监控价格的商品列表 """ resp = {} skus = [] malls = MallInfo.objects.filter(status=1) if malls: for mall in malls: skus.append(mall.sku) return JsonResponse(skus) def bar_base(sku) -> Bar: """ 图表制作 """ x_date = [] y_price = [] mall = MallInfo.objects.filter(sku=sku).last() prices = PriceInfo.objects.filter(sku=sku).order_by('-sid')[:15] if prices: for price in prices[::-1]: x_date.append(price.addtime.strftime('%Y-%m-%d')) y_price.append(price.price) b = (Bar() .add_xaxis(x_date) .add_yaxis("sku:{}".format(sku), y_price, ) .set_global_opts(title_opts=opts.TitleOpts(title="商品历史价格曲线", subtitle="{}".format(mall.name)), xaxis_opts=opts.AxisOpts(axislabel_opts={"interval":"0","rotate":45}), datazoom_opts=[opts.DataZoomOpts(), opts.DataZoomOpts(type_="inside")], ) .dump_options_with_quotes() ) return b class get_mall_price_api(APIView): """ 历史价格获取价格数据api """ def get(self, request, *args, **kwargs): sku = request.GET.get("sku") return JsonResponse(json.loads(bar_base(sku))) class echartsView(APIView): """ 历史价格渲染页面 """ def get(self, request, *args, **kwargs): sku = request.GET.get("sku") p_max = PriceInfo.objects.filter(sku=sku).order_by('-price')[0].price p_min = PriceInfo.objects.filter(sku=sku).order_by('price')[0].price p_now = PriceInfo.objects.filter(sku=sku).last().price n_note = PriceInfo.objects.filter(sku=sku).last().note #多重查询条件 l_note = PriceInfo.objects.filter(sku=sku).exclude(note='无').order_by('-sid') t = Template(open("./templates/echarts.html").read()) c = Context({'sku': sku, 'p_max': p_max, 'p_min': p_min, 'n_note': n_note, 'l_note':l_note, 'p_now':p_now}) html = t.render(c) return HttpResponse(html)echarts.html{message type="success" content="页面模板"/}<!DOCTYPE html> <html> <head> <meta charset="UTF-8"> <title>商品历史价格曲线</title> <script src="https://cdn.bootcss.com/jquery/3.0.0/jquery.min.js"></script> <script type="text/javascript" src="https://assets.pyecharts.org/assets/echarts.min.js"></script> </head> <body> <div style="position:absolute; top:10; left:10; width:1200px; height:95%;"> <div id="line" style="width:1100px;height:93%"></div> <div style="margin-top:20px"> <span>历史高价: <a style='color:red'>{{p_max}}</a></span> <span>历史低价: <a style='color:green'>{{p_min}}</a> </span> <span>当前价格: <a style='color:orange'>{{p_now}}</a> </span> <span>最新活动: <a style='color:blue'>{{n_note}}</a> </span> </div> </div> <div style="margin-left:1020px; height:865px; overflow:auto;"> <table style="maxHeight:1200px;" border="1" cellpadding="3" cellspacing="0"> <caption> 历史活动 </caption> <tr> <td>日期</td><td>内容</td> <tr> {% for i in l_note %} <tr> <td>{{ i.addtime }}</td> <td> {{ i.note }}</td> </tr> {% endfor %} </table> </div> <script> var chart = echarts.init(document.getElementById('line'), 'white', {renderer: 'canvas'}); $( function () { fetchData(chart); } ); function fetchData() { $.ajax({ type: "GET", url: "/mall/getprices/?sku={{sku}}", dataType: 'json', success: function (result) { chart.setOption(result.data); } }); } </script> </body> </html>{card-default label="历史价格" width="95%"}{/card-default}feadper 价格采集crawl_mall_spider.py隐藏内容，请前往内页查看详情

wordpress商品信息整理并通过Django管理 前言{callout color="#f0ad4e"}通过上一篇文章： wordpress直接通过数据库导出文章标题、分类信息到xls表格 ，将数据导入到excel表格中了。在处理的过程中发现仍然不是很方便，一是数据量有点大，每次打开更新需要的的时间有点久。另外无法统计分析。于是想把它输入到自己的数据库中，方便处理查看查询。{/callout}Django添加新模块resource{message type="success" content="models.py 建立数据库模型"/}from django.db import models #状态 U = 0 Y = 1 N = 2 STATUS_CHOICE = { U: '未处理', Y: '可用', N: '不可用', } # Create your models here. class Resource(models.Model): ''' 任务模型类 ''' id = models.AutoField(primary_key=True) rid = models.CharField(max_length=16, verbose_name='编号', default='0') rtitle = models.CharField(max_length=256, verbose_name='标题') rcontent = models.TextField(max_length=2048*8, verbose_name='说明') rparent1 = models.CharField(max_length=128, verbose_name='一级分类', default='0') rparent2 = models.CharField(max_length=128, verbose_name='二级分类', default='1') rurl = models.CharField(max_length=256, verbose_name='下载链接', default='0') rpwd = models.CharField(max_length=256, verbose_name='解压密码', default='0') status_choices = ((k, v) for k,v in STATUS_CHOICE.items()) rstatus = models.SmallIntegerField(default='0', choices=status_choices, verbose_name='状态') # admin显示订单的id def __str__(self): return self.rtitle class Meta: db_table = 'code_resource' verbose_name = '源码资源' verbose_name_plural = '源码资源'{message type="success" content="更新数据库"/}# 修改项目settings.py,添加模块 INSTALLED_APPS = [ ... 'resource', ... ], # 更新 python3 manage.py makemigrations python3 manage.py migrate{message type="success" content="查看建库语句"/}show create table code_resource;待处理资源wordpress数据库操作{message type="success" content="创建code_resource表"/}CREATE TABLE `code_resource` ( `id` int(11) NOT NULL AUTO_INCREMENT, `rid` varchar(16) COLLATE utf8_bin NOT NULL, `rtitle` varchar(256) COLLATE utf8_bin NOT NULL, `rcontent` longtext COLLATE utf8_bin NOT NULL, `rparent1` varchar(128) COLLATE utf8_bin NOT NULL, `rparent2` varchar(128) COLLATE utf8_bin NOT NULL, `rurl` varchar(256) COLLATE utf8_bin NOT NULL, `rpwd` varchar(256) COLLATE utf8_bin NOT NULL, `rstatus` smallint(6) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=60031 DEFAULT CHARSET=utf8 COLLATE=utf8_bin隐藏内容，请前往内页查看详情{message type="success" content="数据迁移处理"/}# 导出 mysqldump -uroot -pxxxx -h 127.0.0.1 yonp code_resource > code_resource.sql # 导入 mysql -uroot -pxxxx taskmonitor < code_resource.sql数据展示{message type="success" content="修改Django项目resource模块的admin.py文件"/}隐藏内容，请前往内页查看详情{card-default label="效果" width="90%"}{/card-default}FAQdjango 数据库更新抱错后，重新执行# 确保存在文件 resource/migrations/__init__.py # 登录数据库,清理resource模块记录 select * from django_migrations;