{"id":795,"date":"2024-11-02T15:36:02","date_gmt":"2024-11-02T07:36:02","guid":{"rendered":"https:\/\/abytelalala.cn\/?p=795"},"modified":"2024-11-11T19:39:25","modified_gmt":"2024-11-11T11:39:25","slug":"%e5%88%a9%e7%94%a8pydoop%e5%b0%86mysql%e5%af%bc%e5%85%a5hdfs","status":"publish","type":"post","link":"https:\/\/abytelalala.cn\/index.php\/2024\/11\/02\/%e5%88%a9%e7%94%a8pydoop%e5%b0%86mysql%e5%af%bc%e5%85%a5hdfs\/","title":{"rendered":"\u5229\u7528pydoop\u5c06mysql\u5bfc\u5165hdfs"},"content":{"rendered":"\n<p>\u9996\u5148\u914d\u7f6e\u73af\u5883\u53d8\u91cfnano ~\/.bashrc\uff0c\u6dfb\u52a0\u4e0b\u9762\u8fd9\u51e0\u884c<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>export HADOOP_HOME=\/usr\/local\/hadoop\nexport PATH=$HADOOP_HOME\/bin:$PATH\nexport HADOOP_CLASSPATH=$HADOOP_HOME\/etc\/hadoop:$HADOOP_HOME\/share\/hadoop\/common\/<em>:$HADOOP_HOME\/share\/hadoop\/mapreduce\/<\/em>\nexport LD_LIBRARY_PATH=$HADOOP_HOME\/lib\/native<\/code><\/pre>\n\n\n\n<p>source ~\/.bashrc<\/p>\n\n\n\n<p>\u4f8b\u5982\u670d\u52a1\u5668\u8fd0\u884c\u5982\u4e0bapi<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>from flask import Flask, request, jsonify\nimport pandas as pd\nfrom pydoop.hdfs import hdfs\nimport os\n\napp = Flask(__name__)\n\n@app.route('\/upload_csv', methods=&#91;'POST'])\ndef upload_csv():\n    # \u4ece\u8bf7\u6c42\u4e2d\u83b7\u53d6\u6570\u636e\n    data = request.json\n    local_file_path = data&#91;'local_file_path']  # \u672c\u5730\u6587\u4ef6\u8def\u5f84\n    hdfs_path = data&#91;'hdfs_path']  # HDFS \u8def\u5f84\n\n    try:\n        # \u68c0\u67e5\u6587\u4ef6\u662f\u5426\u5b58\u5728\n        if not os.path.exists(local_file_path):\n            return jsonify({\"error\": \"\u672c\u5730\u6587\u4ef6\u4e0d\u5b58\u5728\"}), 400\n        df = pd.read_csv(local_file_path)\n        df.to_csv(hdfs_path, index=False)\n        \n        return jsonify({\"message\": \"\u6587\u4ef6\u5df2\u6210\u529f\u4e0a\u4f20\u5230 HDFS\", \"hdfs_path\": hdfs_path}), 200\n\n    except Exception as e:\n        return jsonify({\"error\": str(e)}), 400\n\nif __name__ == '__main__':\n    app.run(host='0.0.0.0', port=5000)\n<\/code><\/pre>\n\n\n\n<p>\u8f93\u5165\u5982\u4e0b\u547d\u4ee4hadoop fs -ls \/user\/hadoop\/\u67e5\u770bhdfs\u6587\u4ef6\u7cfb\u7edf<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>root@hadoop01:~# hadoop fs -ls \/user\/hadoop\/<br>Found 2 items<br>-rw-r--r-- 3 root supergroup 32 2024-11-02 13:36 \/user\/hadoop\/1.csv<br>-rw-r--r-- 3 root supergroup 32 2024-11-02 13:39 \/user\/hadoop\/1.xlsx<br>root@hadoop01:~#<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u9996\u5148\u914d\u7f6e\u73af\u5883\u53d8\u91cfnano ~\/.bashrc\uff0c\u6dfb\u52a0\u4e0b\u9762\u8fd9\u51e0\u884c source ~\/.bashrc \u4f8b\u5982\u670d\u52a1\u5668\u8fd0\u884c [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[7],"tags":[11],"class_list":["post-795","post","type-post","status-publish","format-standard","hentry","category-7","tag-hadoop"],"_links":{"self":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/795","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/comments?post=795"}],"version-history":[{"count":12,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/795\/revisions"}],"predecessor-version":[{"id":842,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/795\/revisions\/842"}],"wp:attachment":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/media?parent=795"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/categories?post=795"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/tags?post=795"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}