{"id":252,"date":"2024-06-27T12:06:18","date_gmt":"2024-06-27T04:06:18","guid":{"rendered":"https:\/\/abytelalala.cn\/?p=252"},"modified":"2024-06-30T12:13:59","modified_gmt":"2024-06-30T04:13:59","slug":"%e5%9f%ba%e4%ba%8edocker%e7%9a%84hadoophbasespark%e4%b8%80%e9%94%ae%e9%83%a8%e7%bd%b2","status":"publish","type":"post","link":"https:\/\/abytelalala.cn\/index.php\/2024\/06\/27\/%e5%9f%ba%e4%ba%8edocker%e7%9a%84hadoophbasespark%e4%b8%80%e9%94%ae%e9%83%a8%e7%bd%b2\/","title":{"rendered":"\u57fa\u4e8edocker\u7684hadoop,hbase,spark\u4e00\u952e\u90e8\u7f72"},"content":{"rendered":"\n

\u5982\u679c\u524d\u9762\u6ca1\u642d\u5efadocker\uff0c\u8be6\u60c5\u8bf7\u770b\u8fd9\u7bc7\u6587\u7ae0\uff1ahttps:\/\/abytelalala.cn\/index.php\/2024\/06\/24\/ubentu%e4%bb%8e%e9%9b%b6%e5%bc%80%e5%a7%8b%e9%83%a8%e7%bd%b2docker\/<\/a><\/p>\n\n\n\n

nano \/etc\/docker\/daemon.json\n\u5199\u5165\u5982\u4e0b\u4e09\u884c\u5185\u5bb9\n{\n  \"registry-mirrors\": [\"https:\/\/docker.foreverlink.love\/\"]\n}\n\nservice docker restart\ndocker info\/\/\u9a8c\u8bc1\u6210\u529f\u66ff\u6362\u6210\u529f\u6ca1\u6709\ndocker pull ubuntu:latest\n\ndocker run -it ubuntu:latest\n\napt-get update\napt update\napt upgrade<\/code><\/pre>\n\n\n\n
net-tools<\/h2>\n\n\n\n
apt install net-tools<\/code><\/pre>\n\n\n\n
java<\/h2>\n\n\n\n
apt-get install openjdk-8-jdk<\/code><\/pre>\n\n\n\n
\u5b89\u88c5\u5305<\/h2>\n\n\n\n
\u4e0b\u8f7d\u5b89\u88c5\u5305
hadoop-3.2.2.tar.gz\uff0c
hbase-2.5.8-bin.tar.gz\uff0c
apache-zookeeper-3.7.2-bin.tar.gz
\u62d6\u52a8\u5230mobaXterm\u7684\u865a\u62df\u673a\u9875\u9762\u76ee\u5f55<\/p>\n\n\n\n
\u89e3\u538b\u79fb\u52a8\u7b49\u64cd\u4f5c<\/h2>\n\n\n\n
\u63a5\u4e0b\u6765\u64cd\u4f5c\u9700\u8981\u518d\u5f00\u4e00\u4e2a\u65b0\u7ec8\u7aef\uff0c\u4fdd\u7559\u4e4b\u524d\u7684\u7ec8\u7aef<\/p>\n\n\n\n
su -\ndocker ps \/\/\u67e5\u770b\u6307\u5b9a\u7684\u5bb9\u5668id\ndocker cp \/home\/cust\/hadoop-3.2.2.tar.gz <\u6307\u5b9a\u5bb9\u5668id>:\/usr\/local\/ \/\/\u79fb\u52a8\u5230docker\u91cc\ndocker cp \/home\/cust\/hbase-2.5.8-bin.tar.gz <\u6307\u5b9a\u5bb9\u5668id>:\/usr\/local\/\ndocker cp \/home\/cust\/apache-zookeeper-3.7.2-bin.tar.gz <\u6307\u5b9a\u5bb9\u5668id>:\/usr\/local\/ \/\/\u8fd9\u4e24\u4e2a\u90fd\u540c\u7406<\/code><\/pre>\n\n\n\n
\u7136\u540e\u8fdb\u5165\u539f\u5bb9\u5668\u7684\u7ec8\u7aef\uff0c\u8f93\u5165\u5982\u4e0b\u547d\u4ee4<\/p>\n\n\n\n
cd \/usr\/local\nls\n\/\/\u67e5\u770b\u662f\u5426\u8f6c\u79fb\u5230\u8fd9\u91cc\u6765\u4e86\n\ntar -zxvf hadoop-3.2.2.tar.gz\nmv hadoop-3.2.2 \/usr\/local\/hadoop\ntar -zxvf apache-zookeeper-3.7.2-bin.tar.gz\nmv apache-zookeeper-3.7.2-bin \/usr\/local\/zookeeper\ntar -zxvf hbase-2.5.8-bin.tar.gz\nmv hbase-2.5.8 \/usr\/local\/hbase \/\/\u5177\u4f53\u4f60\u4eec\u6587\u4ef6\u89e3\u538b\u540e\u662f\u4ec0\u4e48\u540d\u5b57\uff0c\u5177\u4f53\u7528ls\u547d\u4ee4\u6765\u67e5\u770b\u4fee\u6539\n<\/code><\/pre>\n\n\n\n
\u6dfb\u52a0\u73af\u5883\u53d8\u91cf<\/h2>\n\n\n\n
apt install nano\nnano \/etc\/profile<\/code><\/pre>\n\n\n\n
\u5728\/etc\/profile\u6dfb\u52a0\u5982\u4e0b\u5185\u5bb9<\/h3>\n\n\n\n
JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64\nHADOOP_HOME=\/usr\/local\/hadoop\nexport PATH=$JAVA_HOME\/bin:$PATH\nexport PATH=$PATH:$HADOOP_HOME\/bin:$HADOOP_HOME\/sbin\n#root\u542f\u52a8\uff0c\u6dfb\u52a0\u4ee5\u4e0b\u73af\u5883\nexport HDFS_NAMENODE_USER=root\nexport HDFS_DATANODE_USER=root\nexport HDFS_SECONDARYNAMENODE_USER=root\nexport YARN_RESOURCEMANAGER_USER=root\nexport YARN_NODEMANAGER_USER=root\nexport ZOOKEEPER_HOME=\/usr\/local\/zookeeper\nexport PATH=$PATH:$ZOOKEEPER_HOME\/bin\nexport HBASE_HOME=\/usr\/local\/hbase\nexport PATH=$PATH:$HBASE_HOME\/bin<\/code><\/pre>\n\n\n\n
ssh\u514d\u5bc6\u767b\u5165<\/h2>\n\n\n\n
apt-get update\napt-get install systemd\napt-get update --fix-missing\napt-get install openssh-server\nssh-keygen -t rsa \/\/\u9047\u89c1\u8f93\u5165\u5bc6\u7801\u4ec0\u4e48\u7684\u90fd\u56de\u8f66\ncat ~\/.ssh\/id_rsa.pub >> ~\/.ssh\/authorized_keys\n\nservice ssh start\nservice ssh status\n\/\/\u68c0\u67e5\u4e00\u4e0bsshd\u662f\u5426\u5f00\u542f\nssh localhost\n\/\/\u8fd0\u884c\u5b8c\u6210\u4f1a\u5f39\u51fa\u4e00\u4e2a\u95ee\u4f60\u7b2c\u4e00\u6b21\u767b\u9646\u662f\u5426\u7684\u95ee\u9898\n\/\/\u4f60\u8981\u56de\u7b54yes\uff0c\u4e0d\u8981\u53ea\u56de\u7b54\u4e00\u4e2ay<\/code><\/pre>\n\n\n\n
hadoop\u914d\u7f6e\u6587\u4ef6<\/h2>\n\n\n\n
\u5728\/usr\/local\/hadoop\/etc\/hadoop\/\u6587\u4ef6\u5939\u4e0b<\/p>\n\n\n\n
hadoop-env.sh\u6587\u4ef6<\/h3>\n\n\n\n
\u52a0\u5165\uff1a
\u200b
# \u663e\u5f0f\u58f0\u660ejava\u8def\u5f84
export JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64\/<\/code><\/pre>\n\n\n\n
core-site.xml\u6587\u4ef6<\/h3>\n\n\n\n
<configuration>
    <property>
        <name>fs.defaultFS<\/name>
        <value>hdfs:\/\/hadoop01:9000<\/value>
    <\/property>
    <!-- \u6307\u5b9ahadoop\u8fd0\u884c\u65f6\u4ea7\u751f\u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84 -->
    <property>
        <name>hadoop.tmp.dir<\/name>
        <!-- \u914d\u7f6e\u5230hadoop\u76ee\u5f55\u4e0btemp\u6587\u4ef6\u5939 -->
        <value>file:\/usr\/local\/hadoop\/tmp<\/value>
    <\/property>
    <property>
        <name>hadoop.proxyuser.root.hosts<\/name>
        <value>*<\/value>
    <\/property>
    <property>
        <name>hadoop.proxyuser.root.groups<\/name>
        <value>*<\/value>
    <\/property>
<\/configuration><\/code><\/pre>\n\n\n\n
hdfs-site.xml\u6587\u4ef6<\/h3>\n\n\n\n
<configuration>
    <property>
    <!--\u6307\u5b9ahdfs\u4fdd\u5b58\u6570\u636e\u526f\u672c\u7684\u6570\u91cf\uff0c\u5305\u62ec\u81ea\u5df1\uff0c\u9ed8\u8ba4\u4e3a3-->
    <!--\u4f2a\u5206\u5e03\u5f0f\u6a21\u5f0f\uff0c\u6b64\u503c\u5fc5\u987b\u4e3a1-->
        <name>dfs.replication<\/name>
        <value>3<\/value>
    <\/property>
    <property>
        <name>dfs.namenode.name.dir<\/name>
        <!-- name node \u5b58\u653e name table \u7684\u76ee\u5f55 -->
        <value>file:\/usr\/local\/hadoop\/tmp\/hdfs\/name<\/value>
    <\/property>
    <property>
        <name>dfs.namenode.secondary.http-address<\/name>
        <value>hadoop02:50090<\/value>
    <\/property>
    <property>
        <name>dfs.datanode.data.dir<\/name>
        <!-- data node \u5b58\u653e\u6570\u636e block \u7684\u76ee\u5f55 -->
        <value>file:\/usr\/local\/hadoop\/tmp\/hdfs\/data<\/value>
    <\/property>
    <property>  
        <name>dfs.permissions<\/name>
        <value>true<\/value>
    <\/property>
    <property>
        <name>dfs.permissions.enabled<\/name>
        <value>false<\/value>
    <\/property>
<\/configuration><\/code><\/pre>\n\n\n\n
mapred-site.xml \u6587\u4ef6<\/h3>\n\n\n\n
<configuration>
    <property>
    <!--\u6307\u5b9amapreduce\u8fd0\u884c\u5728yarn\u4e0a-->
        <name>mapreduce.framework.name<\/name>
        <value>yarn<\/value>
    <\/property>
    <property>
        <name>yarn.app.mapreduce.am.env<\/name>
        <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}<\/value>
    <\/property>
    <property>
        <name>mapreduce.map.env<\/name>
        <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}<\/value>
    <\/property>
    <property>
        <name>mapreduce.reduce.env<\/name>
        <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}<\/value>
    <\/property>
<\/configuration><\/code><\/pre>\n\n\n\n
yarn-site.xml \u6587\u4ef6<\/h3>\n\n\n\n
<configuration>
    <property>
        <name>yarn.resourcemanager.hostname<\/name>
        <value>hadoop01<\/value>
    <\/property>
    <property>
    <!--NodeManager\u83b7\u53d6\u6570\u636e\u7684\u65b9\u5f0f-->
      <name>yarn.nodemanager.aux-services<\/name>
      <value>mapreduce_shuffle<\/value>
    <\/property>
<\/configuration><\/code><\/pre>\n\n\n\n
workers\u6587\u4ef6<\/h3>\n\n\n\n
hadoop01
hadoop02
hadoop03<\/code><\/pre>\n\n\n\n
zookeeper\u914d\u7f6e<\/h2>\n\n\n\n
\u5728\/usr\/local\/zookeeper\/conf\/\u6587\u4ef6\u5939\u4e0b<\/p>\n\n\n\n
zoo.cfg\u6587\u4ef6<\/h3>\n\n\n\n
tickTime=2000
\u200b
initLimit=10
\u200b
syncLimit=5
\u200b
dataDir=\/usr\/local\/zookeeper\/data
\u200b
clientPort=2181
\u200b
server.1=hadoop01:2888:3888
server.2=hadoop02:2888:3888
server.3=hadoop03:2888:3888<\/pre>\n\n\n\n
myid\u6587\u4ef6<\/h3>\n\n\n\n
\u5728\u96c6\u7fa4\u6bcf\u53f0\u673a\u5668\u7684 dataDir \u76ee\u5f55\u4e0b\u5efa\u7acb myid \u6587\u4ef6\uff0c\u6587\u4ef6\u5185\u5bb9\u4e0e server.x \u7684 x \u503c\u4e00\u81f4\uff0c\u6211\u4eec\u8fd9\u91cc\u662f\u5728\u955c\u50cf\u4fee\u6539\u914d\u7f6e\uff0c\u6240\u4ee5\u5148\u90fd\u5f04\u62101\uff0c\u4e4b\u540e\u518d\u6539\u3002<\/p>\n\n\n\n
cd \/usr\/local\/zookeeper\nmkdir data\nnano myid<\/code><\/pre>\n\n\n\n
myid\u5185\u5bb9\u5982\u4e0b\uff1a<\/p>\n\n\n\n
1<\/pre>\n\n\n\n
hbase\u914d\u7f6e<\/h2>\n\n\n\n
\u5728\/usr\/local\/hbase\/conf\/\u6587\u4ef6\u5939\u4e0b<\/p>\n\n\n\n
hbase-site.xml\u6587\u4ef6<\/h3>\n\n\n\n
<configuration>
     <property>
        <name>hbase.rootdir<\/name>
        <value>hdfs:\/\/hadoop01:9000\/hbase<\/value>
    <\/property>
    <property>
        <name>hbase.zookeeper.quorum<\/name>
        <value>hadoop01,hadoop02,hadoop03<\/value>
    <\/property>
    <property>
        <name>hbase.zookeeper.property.clientPort<\/name>
        <value>2181<\/value>
    <\/property>
    <property>
        <name>hbase.cluster.distributed<\/name>
        <value>true<\/value> 
    <\/property>
    <property>
        <name>hbase.zookeeper.property.dataDir<\/name>
        <value>\/usr\/local\/zookeeper\/data<\/value>
    <\/property>
<\/configuration><\/pre>\n\n\n\n
hbase-env.sh\u6587\u4ef6<\/h3>\n\n\n\n
\u52a0\u5165\uff1a
\u200b
export HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP=true
\u200b
export JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64
export HBASE_CLASSPATH=\/usr\/local\/hbase\/conf
export HBASE_MANAGES_ZK=flase<\/pre>\n\n\n\n
backup-master\u6587\u4ef6<\/h3>\n\n\n\n
hadoop01<\/pre>\n\n\n\n
regionservers\u6587\u4ef6<\/h3>\n\n\n\n
hadoop01
hadoop02
hadoop03<\/pre>\n\n\n\n
\u5982\u679c\u6ca1\u642d\u5efahbase\uff0c\u8be6\u60c5\u8bf7\u770b\u8fd9\u7bc7https:\/\/abytelalala.cn\/index.php\/2024\/06\/26\/%e5%9f%ba%e4%ba%8eubentudocker%e6%90%ad%e5%bb%bahadoop%ef%bc%8czookeeper%ef%bc%8chbase\/<\/a><\/p>\n\n\n\n
\u5b89\u88c5\u5305<\/h2>\n\n\n\n
\u4e0b\u8f7d\u5b89\u88c5\u5305
scala-2.12.13.tgz
spark-3.0.0-bin-hadoop3.2.tgz
jackson-databind-2.10.1.jar
\/\/\u4e0b\u8f7d\u94fe\u63a5Central Repository: com\/fasterxml\/jackson\/core\/jackson-databind\/2.10.1 (maven.org)<\/a>
scala-2.12.13.tgz
spark-3.0.0-bin-hadoop3.2.tgz
\u62d6\u52a8\u5230mobaXterm\u7684\u865a\u62df\u673a\u9875\u9762\/home\/cust\u76ee\u5f55<\/p>\n\n\n\n
\u5728\u5bb9\u5668hadoop01\u91cc\u8f93\u5165\u4e0b\u9762\u4e24\u884c\u547d\u4ee4<\/p>\n\n\n\n
su -\nmkdir \/jara<\/code><\/pre>\n\n\n\n
jackson-databind-2.10.1.jar
\u81ea\u5df1\u65b0\u5efajara\u6587\u4ef6\u5939\uff0c\u5c06\u8fd9\u4e2ajar\u62d6\u52a8\u5230mobaXterm\u7684\u865a\u62df\u673a\u9875\u9762\/home\/cust\/jara\u76ee\u5f55<\/p>\n\n\n\n
\u79fb\u52a8\u5230hadoop01\u4e0a<\/p>\n\n\n\n
su - \ndocker ps \/\/\u67e5\u770b\u6307\u5b9a\u7684\u5bb9\u5668id \ndocker cp \/home\/cust\/jara\/jackson-databind-2.10.1.jar <\u6307\u5b9a\u5bb9\u5668id>:\/jara\/ \/\/\u79fb\u52a8\u5230docker\u91cc\ndocker cp \/home\/cust\/scala-2.12.13.tgz <\u6307\u5b9a\u5bb9\u5668id>:\/usr\/local\/ \/\/\u79fb\u52a8\u5230docker\u91cc\ndocker cp \/home\/cust\/spark-3.0.0-bin-hadoop3.2.tgz <\u6307\u5b9a\u5bb9\u5668id>:\/usr\/local\/ \/\/\u79fb\u52a8\u5230docker\u91cc<\/code><\/pre>\n\n\n\n
\u8fdb\u5165hadoop01\u7ec8\u7aef<\/p>\n\n\n\n
docker exec -it hadoop01 bash<\/code><\/pre>\n\n\n\n
\u914d\u7f6espark<\/h2>\n\n\n\n
cd \/usr\/local\ntar -zxvf spark-3.0.0-bin-hadoop3.2.tgz\nmv spark-3.0.0-bin-hadoop3.2  \/usr\/local\/spark\ncd \/usr\/local\/spark\/conf\nmv spark-defaults.conf.template spark-defaults.conf\nmv slaves.template slaves<\/code><\/pre>\n\n\n\n
spark-defaults.conf<\/h3>\n\n\n\n
nano spark-defaults.conf\n\n        spark.driver.extraClassPath        \/jara\/jackson-databind-2.10.1.jar\n        spark.executor.extraClassPath      \/jara\/jackson-databind-2.10.1.jar<\/code><\/pre>\n\n\n\n
slaves<\/h3>\n\n\n\n
nano slaves \n\thadoop01\n\thadoop02\n\thadoop03<\/code><\/pre>\n\n\n\n
spark-env.sh<\/h3>\n\n\n\n
nano spark-env.sh\n\tSPARK_MASTER_HOST=hadoop01\n        SPARK_MASTER_PORT=7077\n\texport JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64\n\texport HADOOP_HOME=\/usr\/local\/hadoop\/\n\texport SPARK_CLASSPATH=\/usr\/local\/hbase\/lib\/*\n\texport SCALA_HOME=\/usr\/local\/spark\/\n\texport HADOOP_CONF_DIR=\/usr\/loacl\/hadoop\/etc\/hadoop\/\n\texport SPARK_WORKER_MERMORY=8G<\/code><\/pre>\n\n\n\n
spark-config.sh<\/h3>\n\n\n\n
cd ..\/sbin\/\nnano spark-config.sh \n\texport JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64<\/code><\/pre>\n\n\n\n
\u73af\u5883\u53d8\u91cf\u914d\u7f6e<\/h2>\n\n\n\n
cd \/usr\/local\/\ntar -zxvf scala-2.12.13.tgz\nmv  scala-2.12.13  \/usr\/local\/scala\nnano \/etc\/profile\n\texport SCALA_HOME=\/usr\/local\/scala\n\texport PATH=.:$SCALA_HOME\/bin:$PATH\n\texport SPARK_HOME=\/usr\/local\/spark\n\texport PATH=$PATH:$SPARK_HOME\/bin:$SPARK_HOME\/sbin\nsource \/etc\/profile<\/code><\/pre>\n\n\n\n
\u63d0\u4ea4\u955c\u50cf<\/h2>\n\n\n\n
docker commit hadoop01 sparknew<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
\u5982\u679c\u524d\u9762\u6ca1\u642d\u5efadocker\uff0c\u8be6\u60c5\u8bf7\u770b\u8fd9\u7bc7\u6587\u7ae0\uff1ahttps:\/\/abytelalala.cn\/index.php […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[8],"tags":[10,11,15,14,16],"class_list":["post-252","post","type-post","status-publish","format-standard","hentry","category-8","tag-docker","tag-hadoop","tag-hbase","tag-spark","tag-zookeeper"],"_links":{"self":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/252","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/comments?post=252"}],"version-history":[{"count":9,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/252\/revisions"}],"predecessor-version":[{"id":263,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/posts\/252\/revisions\/263"}],"wp:attachment":[{"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/media?parent=252"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/categories?post=252"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/abytelalala.cn\/index.php\/wp-json\/wp\/v2\/tags?post=252"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}