{"id":327,"date":"2024-07-16T11:14:54","date_gmt":"2024-07-16T03:14:54","guid":{"rendered":"https:\/\/abytelalala.cn\/?p=327"},"modified":"2024-07-16T11:14:54","modified_gmt":"2024-07-16T03:14:54","slug":"%e6%9c%8d%e5%8a%a1%e5%99%a8%e4%b8%8aspark%e4%bd%bf%e7%94%a8python%e8%84%9a%e6%9c%ac%e5%b0%81%e8%a3%85%e4%b8%baapi","status":"publish","type":"post","link":"https:\/\/abytelalala.cn\/index.php\/2024\/07\/16\/%e6%9c%8d%e5%8a%a1%e5%99%a8%e4%b8%8aspark%e4%bd%bf%e7%94%a8python%e8%84%9a%e6%9c%ac%e5%b0%81%e8%a3%85%e4%b8%baapi\/","title":{"rendered":"\u670d\u52a1\u5668\u4e0aspark\u4f7f\u7528python\u811a\u672c\u5c01\u88c5\u4e3aapi"},"content":{"rendered":"\n<p>\u6709\u4e00\u4e2a\u9700\u6c42\u662f\u5b9e\u73b0\u628apython\u811a\u672c\u63d0\u4ea4\u4e3aspark\u4efb\u52a1\u5c01\u88c5\u4e3aapi\uff0c\u8fd9\u91cc\u601d\u8def\u662f\u7528\u4e00\u4e2aflask\u6846\u67b6\uff0c\u4f20\u9012json\u53c2\u6570\u662f\u811a\u672c\u6587\u4ef6\u8def\u5f84\u548c\u540d\u79f0\uff0c\u8fd9\u6837api\u5c31\u77e5\u9053\u8981\u6267\u884c\u54ea\u4e2a\u811a\u672c\u4e86\u3002\u5173\u95ed\u4e5f\u662f\u4e00\u6837\uff0c\u7528json\u53c2\u6570\u4f20\u9012\u4e00\u4e2a\u5173\u95ed\u52a8\u4f5c\uff0capi\u670d\u52a1\u5c31\u5173\u95ed\u4e86\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">api.py<\/h2>\n\n\n\n<pre class=\"wp-block-code\"><code>from flask import Flask, request, jsonify\nimport os\nimport subprocess\nimport signal\nimport sys\napp = Flask(<strong>name<\/strong>)\n@app.route('\/shutdown', methods=&#91;'POST'])\ndef shutdown():\n\u00a0 \u00a0 if request.method == 'POST':\n\u00a0 \u00a0 \u00a0 \u00a0 shutdown_server()\n\u00a0 \u00a0 \u00a0 \u00a0 return jsonify({\"message\": \"Server is shutting down\u2026\"}), 200\ndef shutdown_server():\n\u00a0 \u00a0 pid = os.getpid()\n\u00a0 \u00a0 os.kill(pid, signal.SIGINT)\n@app.route('\/submit_spark_job', methods=&#91;'POST'])\ndef submit_spark_job():\n\u00a0 \u00a0 data = request.get_json()\n\u00a0 \u00a0 file_path = data&#91;'file_path']\n\u00a0 \u00a0 file_name = data&#91;'file_name']\n\u00a0 \u00a0 # \u68c0\u67e5\u6587\u4ef6\u662f\u5426\u5b58\u5728\n\u00a0 \u00a0 if not os.path.exists(file_path):\n\u00a0 \u00a0 \u00a0 \u00a0 return jsonify({\"error\": \"\u6587\u4ef6\u4e0d\u5b58\u5728\"}), 400\n\u00a0 \u00a0 # \u6784\u5efa\u5b8c\u6574\u7684\u6587\u4ef6\u8def\u5f84\n\u00a0 \u00a0 full_file_path = os.path.join(file_path, file_name)\n\u00a0 \u00a0 # \u63d0\u4ea4Spark\u4efb\u52a1\n\u00a0 \u00a0 try:\n\u00a0 \u00a0 \u00a0 \u00a0 result = subprocess.run(&#91;'\/usr\/local\/spark\/bin\/spark-submit', full_file_path], capture_output=True, text=True)\n\u00a0 \u00a0 \u00a0 \u00a0 return jsonify({\"result\": 'Successfully executed Spark task'}), 200\n\u00a0 \u00a0 except Exception as e:\n\u00a0 \u00a0 \u00a0 \u00a0 return jsonify({\"error\": str(e)}), 500\ndef signal_handler(sig, frame):\n\u00a0 \u00a0 print('\u6536\u5230\u5173\u95ed\u4fe1\u53f7\uff0c\u6b63\u5728\u5173\u95ed\u670d\u52a1\u5668\u2026')\n\u00a0 \u00a0 sys.exit(0)\nif name == 'main':\n\u00a0 \u00a0 signal.signal(signal.SIGINT, signal_handler)\n\u00a0 \u00a0 signal.signal(signal.SIGTERM, signal_handler)\n\u00a0 \u00a0 app.run(host='0.0.0.0', debug=True, port=8084)<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">\u8fd0\u884capi.py<\/h2>\n\n\n\n<p>\u5728api.py\u76f8\u5e94\u76ee\u5f55\u4e0b\u6267\u884c\u8fd9\u4e2a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>python3 api.py<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">\u67e5\u770bpython\u8fdb\u7a0b\u90fd\u6709\u4ec0\u4e48<\/h2>\n\n\n\n<pre class=\"wp-block-code\"><code>ps -ef | grep python | grep -v grep<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">post\u6d4b\u8bd5<\/h2>\n\n\n\n<p>\u8fd9\u91cc\u6211\u4eec\u53ef\u4ee5\u9009\u62e9burp\u6765\u8fdb\u884cpost\u6d4b\u8bd5\uff0c\u5f53\u7136\u7528postman\u4e5f\u53ef\u4ee5\uff0c\u5173\u95ed\u547d\u4ee4\u7684post\u683c\u5f0f\u5982\u4e0b\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>POST \/shutdown HTTP\/1.1\nHost: 192.168.1.110:8084\nUser-Agent: Mozilla\/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/115.0.5790.171 Safari\/537.36\nAccept: application\/json\nAccept-Encoding: gzip, deflate\nAccept-Language: zh-CN,zh;q=0.9\nConnection: close\nContent-Type: application\/json\nContent-Length: 21\n\n{\n\u00a0 \"action\": \"shutdown\"\n}<\/code><\/pre>\n\n\n\n<p>\u8fd0\u884c\u6307\u5b9a\u811a\u672c\u7684post\u683c\u5f0f\u5982\u4e0b\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>POST \/submit_spark_job HTTP\/1.1\nHost: 192.168.1.110:8084\nUser-Agent: Mozilla\/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/115.0.5790.171 Safari\/537.36\nAccept: application\/json\nAccept-Encoding: gzip, deflate\nAccept-Language: zh-CN,zh;q=0.9\nConnection: close\nContent-Type: application\/json\nContent-Length: 57\n\n{\n  \"file_path\": \"\/py\/\",\n  \"file_name\": \"example.py\"\n}<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u6709\u4e00\u4e2a\u9700\u6c42\u662f\u5b9e\u73b0\u628apython\u811a\u672c\u63d0\u4ea4\u4e3aspark\u4efb\u52a1\u5c01\u88c5\u4e3aapi\uff0c\u8fd9\u91cc\u601d\u8def\u662f\u7528\u4e00\u4e2aflask\u6846\u67b6\uff0c\u4f20\u9012jso [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[7],"tags":[],"class_list":["post-327","post","type-post","status-publish","format-standard","hentry","category-7"],"_links":{"self":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/327","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/comments?post=327"}],"version-history":[{"count":10,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/327\/revisions"}],"predecessor-version":[{"id":337,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/327\/revisions\/337"}],"wp:attachment":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/media?parent=327"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/categories?post=327"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/tags?post=327"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}