{"id":304,"date":"2024-07-13T14:09:22","date_gmt":"2024-07-13T06:09:22","guid":{"rendered":"https:\/\/abytelalala.cn\/?p=304"},"modified":"2024-07-13T14:09:22","modified_gmt":"2024-07-13T06:09:22","slug":"windows%e6%90%ad%e5%bb%baspark","status":"publish","type":"post","link":"https:\/\/abytelalala.cn\/index.php\/2024\/07\/13\/windows%e6%90%ad%e5%bb%baspark\/","title":{"rendered":"windows\u642d\u5efaspark"},"content":{"rendered":"\n<p>\u9ed8\u8ba4\u914d\u7f6e\u597d\u4e86java\u7684\u73af\u5883\u53d8\u91cf\uff0c\u9ed8\u8ba4\u4e0b\u8f7d\u4e86spark\u7684spark-3.0.0-bin-hadoop3.2.tgz\u5b89\u88c5\u5305\u548cjackson-databind-2.10.1.jar\u5305\uff0c\u9ed8\u8ba4python\u7248\u672c3.6-3.8<br>spark\u5305\u89e3\u538b\u5230D\u76d8\u7684spark\u6587\u4ef6\u5939\u4e0b\uff0c\u540c\u65f6\u5728\u8fd9\u4e2a\u6587\u4ef6\u5939\u4e0b\u521b\u5efajara\u6587\u4ef6\u5939\u548clog\u6587\u4ef6\u5939\uff0c\u5176\u4e2djara\u6587\u4ef6\u5939\u91cc\u9762\u88c5\u4e86jar\u5305\u3002<\/p>\n\n\n\n<p>spark-defaults.conf.template\u6587\u4ef6\u6dfb\u52a0\u5982\u4e0b\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>spark.driver.extraClassPath        D:\\spark\\jara\\jackson-databind-2.10.1.jar\nspark.executor.extraClassPath      D:\\spark\\jara\\jackson-databind-2.10.1.jar\nspark.executor.memory              4G\nspark.driver.memory                2G\nspark.executor.cores               2\nspark.driver.cores                 1\nspark.eventLog.dir                 file:\/\/\/D:\/spark\/log\/event_logs\nspark.history.fs.logDirectory      file:\/\/\/D:\/spark\/log\/history_logs<\/code><\/pre>\n\n\n\n<p>spark-env.sh\u6587\u4ef6\u6dfb\u52a0\u5982\u4e0b\uff0c\u5176\u4e2djava\u73af\u5883\u53d8\u91cf\u9700\u8981\u81ea\u5df1\u67e5\u770b\u81ea\u5df1\u672c\u673a\u7684\u5177\u4f53\u5730\u5740<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>SET JAVA_HOME=D:\\java\\java8\nSET HADOOP_HOME=D:\\hadoop\\bin\\winutils-master\\hadoop-3.0.0\nSET PATH=%PATH%;%HADOOP_HOME%\\bin\n\nSET SPARK_LOG_DIR=D:\\spark\\log\\logs\nSET SPARK_EVENT_LOG_DIR=D:\\spark\\log\\event_logs\nSET SPARK_HISTORY_OPTS=-Dspark.history.fs.logDirectory=D:\\spark\\log\\history_logs\n\nSET SPARK_EXECUTOR_MEMORY=4G\nSET SPARK_DRIVER_MEMORY=2G\nSET SPARK_EXECUTOR_CORES=2\nSET SPARK_DRIVER_CORES=1<\/code><\/pre>\n\n\n\n<p>\u914d\u7f6espark\u548chadoop\u7684\u73af\u5883\u53d8\u91cf\uff0c\u7136\u540e\u6267\u884c<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>D:\\spark\\spark-3.0.0-bin-hadoop3.2\\spark-3.0.0-bin-hadoop3.2\\bin\\spark-submit --master local&#91;*] --executor-memory 1G --total-executor-cores 2 --conf spark.pyspark.python=C:\\Users\\\u5f20\u963f\u6625\\AppData\\Local\\Programs\\Python\\Python36\\python.exe \"D:\\car_data_clean\\hadoop\\hadoop_call_computation\\example.py\"<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u9ed8\u8ba4\u914d\u7f6e\u597d\u4e86java\u7684\u73af\u5883\u53d8\u91cf\uff0c\u9ed8\u8ba4\u4e0b\u8f7d\u4e86spark\u7684spark-3.0.0-bin-hadoop3.2.tg [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[8],"tags":[],"class_list":["post-304","post","type-post","status-publish","format-standard","hentry","category-8"],"_links":{"self":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/304","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/comments?post=304"}],"version-history":[{"count":15,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/304\/revisions"}],"predecessor-version":[{"id":319,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/304\/revisions\/319"}],"wp:attachment":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/media?parent=304"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/categories?post=304"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/tags?post=304"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}