|
林子雨编著《大数据基础编程、实验和案例教程》(教材官网)教材中的代码，在纸质教材中的印刷效果不是很好，可能会影响读者对代码的理解，为了方便读者正确理解代码或者直接拷贝代码用于上机实验，这里提供全书配套的所有代码。
查看教材所有章节的代码
第3章 Hadoop的安装和使用
教材第52页
（温馨提示：代码框上方的复制代码按钮，也就是“两张A4纸图标”，用鼠标点击复制代码按钮，就可以把代码框中的代码复制到粘贴板，粘贴到其他地方。但是，有的浏览器可能不支持该功能）
sudo apt-get update
sudo apt-get install openssh-server
ssh localhost
教材第53页
cd ~/.ssh/ # 若没有该目录，请先执行一次ssh localhost
ssh-keygen -t rsa # 会有提示，都按回车即可
cat ./id_rsa.pub >> ./authorized_keys # 加入授权
教材第54页
sudo apt-get install openjdk-7-jre openjdk-7-jdk
dpkg -L openjdk-7-jdk | grep '/bin/javac'
vim ~/.bashrc
export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
source ~/.bashrc # 使变量设置生效
echo $JAVA_HOME # 检验变量值
java -version
$JAVA_HOME/bin/java -version # 与直接执行java -version一样
教材第55页
sudo apt-get install default-jre default-jdk
vim ~/.bashrc
export JAVA_HOME=/usr/lib/jvm/default-java
source ~/.bashrc # 使变量设置生效
echo $JAVA_HOME # 检验变量值
java -version
$JAVA_HOME/bin/java -version # 与直接执行java -version一样
教材第56页
sudo tar -zxf ~/下载/hadoop-2.7.1.tar.gz -C /usr/local # 解压到/usr/local中
cd /usr/local/
sudo mv ./hadoop-2.7.1/ ./hadoop # 将文件夹名改为hadoop
sudo chown -R hadoop ./hadoop # 修改文件权限
cd /usr/local/hadoop
./bin/hadoop version
cd /usr/local/hadoop
./bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar
cd /usr/local/hadoop
mkdir input
cp ./etc/hadoop/*.xml ./input # 将配置文件复制到input目录下
教材第57页
./bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar grep ./input ./output 'dfs[a-z.]+'
cat ./output/* # 查看运行结果
rm -r ./output
hadoop.tmp.dir
file:/usr/local/hadoop/tmp
Abase for other temporary directories.
fs.defaultFS
hdfs://localhost:9000
教材第58页
dfs.replication
1
dfs.namenode.name.dir
file:/usr/local/hadoop/tmp/dfs/name
dfs.datanode.data.dir
file:/usr/local/hadoop/tmp/dfs/data
教材第59页
cd /usr/local/hadoop
./bin/hdfs namenode -format
cd /usr/local/hadoop
./sbin/start-dfs.sh #start-dfs.sh是个完整的可执行文件，中间没有空格
教材第60页
export HADOOP_HOME=/usr/local/hadoop
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
jps
教材第61页
./sbin/stop-dfs.sh # 关闭
rm -r ./tmp # 删除 tmp 文件，注意这会删除 HDFS中原有的所有数据
./bin/hdfs namenode -format # 重新格式化名称节点
./sbin/start-dfs.sh # 重启
cd /usr/local/hadoop
./bin/hdfs dfs -mkdir -p /user/hadoop
教材第62页
cd /usr/local/hadoop
./bin/hdfs dfs -mkdir input #在HDFS中创建hadoop用户对应的input目录
./bin/hdfs dfs -put ./etc/hadoop/*.xml input #把本地文件复制到HDFS中
./bin/hdfs dfs -ls input
./bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar grep input output 'dfs[a-z.]+'
./bin/hdfs dfs -cat output/*
./bin/hdfs dfs -rm -r output # 删除 output 文件夹
教材第63页
cd /usr/local/hadoop
./sbin/stop-dfs.sh
cd /usr/local/hadoop
./sbin/start-dfs.sh
cd /usr/local/hadoop
mv ./etc/hadoop/mapred-site.xml.template ./etc/hadoop/mapred-site.xml
教材第64页
mapreduce.framework.name
yarn
yarn.nodemanager.aux-services
mapreduce_shuffle
./sbin/start-dfs.sh #启动Hadoop
./sbin/start-yarn.sh #启动YARN
./sbin/mr-jobhistory-daemon.sh start historyserver #开启历史服务器，才能在Web中查看任务运行情况
教材第65页
./sbin/stop-yarn.sh
./sbin/mr-jobhistory-daemon.sh stop historyserver
./sbin/stop-dfs.sh
教材第66页
export PATH=$PATH:/usr/local/hadoop/sbin
export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin
教材第67页
sudo vim /etc/hostname
hadoop@ dblab-VirtualBox:~$
hadoop@ Master:~$
教材第68页
sudo vim /etc/hosts
192.168.1.121 Master
192.168.1.122 Slave1
192.168.1.121 Master
192.168.1.122 Slave1
ping Master -c 3 # 只ping 3次就会停止，否则要按Ctrl+c中断ping命令
ping Slave1 -c 3
教材第69页
cd ~/.ssh # 如果没有该目录，先执行一次ssh localhost
rm ./id_rsa* # 删除之前生成的公匙（如果已经存在）
ssh-keygen -t rsa # 执行该命令后，遇到提示信息，一直按回车就可以
cat ./id_rsa.pub >> ./authorized_keys
scp ~/.ssh/id_rsa.pub hadoop@Slave1:/home/hadoop/
mkdir ~/.ssh # 如果不存在该文件夹需先创建，若已存在，则忽略本命令
cat ~/id_rsa.pub >> ~/.ssh/authorized_keys
rm ~/id_rsa.pub # 用完以后就可以删掉
教材第70页
ssh Slave1
export PATH=$PATH:/usr/local/hadoop/bin:/usr/local/hadoop/sbin
教材第71页
Slave1
fs.defaultFS
hdfs://Master:9000
hadoop.tmp.dir
file:/usr/local/hadoop/tmp
Abase for other temporary directories.
dfs.namenode.secondary.http-address
Master:50090
dfs.replication
1
dfs.namenode.name.dir
file:/usr/local/hadoop/tmp/dfs/name
dfs.datanode.data.dir
file:/usr/local/hadoop/tmp/dfs/data
教材第72页
mapreduce.framework.name
yarn
mapreduce.jobhistory.address
Master:10020
mapreduce.jobhistory.webapp.address
Master:19888
yarn.resourcemanager.hostname
Master
yarn.nodemanager.aux-services
mapreduce_shuffle
cd /usr/local
sudo rm -r ./hadoop/tmp # 删除 Hadoop 临时文件
sudo rm -r ./hadoop/logs/* # 删除日志文件
tar -zcf ~/hadoop.master.tar.gz ./hadoop # 先压缩再复制
cd ~
scp ./hadoop.master.tar.gz Slave1:/home/hadoop
教材第73页
sudo rm -r /usr/local/hadoop # 删掉旧的（如果存在）
sudo tar -zxf ~/hadoop.master.tar.gz -C /usr/local
sudo chown -R hadoop /usr/local/hadoop
hdfs namenode -format
start-dfs.sh
start-yarn.sh
mr-jobhistory-daemon.sh start historyserver
教材第74页
hdfs dfs -mkdir -p /user/hadoop
hdfs dfs -mkdir input
hdfs dfs -put /usr/local/hadoop/etc/hadoop/*.xml input
hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar grep input output 'dfs[a-z.]+'
教材第75页
stop-yarn.sh
stop-dfs.sh
mr-jobhistory-daemon.sh stop historyserver
教材第76页
uname –r
sudo apt-get update
sudo apt-get install apt-transport-https ca-certificates
sudo apt-key adv \
--keyserver hkp://ha.pool.sks-keyservers.net:80 \
--recv-keys 58118E89F3A912897C070ADBF76221572C52609D
echo deb https://apt.dockerproject.org/repo ubuntu-xenial main | sudo tee /etc/apt/sources.list.d/docker.list
sudo apt-get update
教材第77页
apt-cache policy docker-engine
sudo apt-get install docker-engine
sudo service docker start
sudo docker run hello-world
sudo groupadd docker
教材第78页
sudo usermod -aG docker hadoop
docker pull ubuntu
docker images
cd ~
mkdir build
docker run -it -v /home/hadoop/build:/root/build --name ubuntu ubuntu
教材第79页
apt-get update
apt-get install vim
apt-get install ssh
/etc/init.d/ssh start
/etc/init.d/ssh start
ssh-keygen -t rsa #一直按回车键即可
cd ~./ssh
cat id_dsa.pub >> authorized_keys
教材第80页
apt-get install default-jdk
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
export PATH=$PATH:$JAVA_HOME/bin
source ~/.bashrc
docker login
docker ps
教材第81页
docker commit fd1fc69d75a3 ubuntu/jdkinstalled
docker images
docker run -it -v /home/hadoop/build:/root/build --name ubuntu-jdkinstalled ubuntu/jdkinstalled
docker ps
教材第82页
cd /root/build
tar -zxvf hadoop-2.7.1.tar.gz -C /usr/local
cd /usr/local
mv ./hadoop-2.7.1 ./hadoop
cd hadoop
./bin/hadoop version
教材第83页
cd /usr/local/hadoop
vim etc/hadoop/hadoop-env.sh
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
hadoop.tmp.dir
file:/usr/local/hadoop/tmp
Abase for other temporary directories.
fs.defaultFS
hdfs://master:9000
dfs.namenode.name.dir
file:/usr/local/hadoop/namenode_dir
dfs.datanode.data.dir
file:/usr/local/hadoop/datanode_dir
dfs.replication
3
教材第84页
mapreduce.framework.name
yarn
yarn.nodemanager.aux-services
mapreduce_shuffle
yarn.resourcemanager.hostname
master
docker commit a40b99f869ae ubuntu/hadoopinstalled
# 在第一个终端中执行下面命令
docker run -it -h master --name master ubuntu/hadoopinstalled
# 在第二个终端中执行下面命令
docker run -it -h slave01 --name slave01 ubuntu/hadoopinstalled
# 在第三个终端中执行下面命令
docker run -it -h slave02 --name slave02 ubuntu/hadoopinstalled
教材第85页
ssh slave01
ssh slave02
cd /usr/local/hadoop
vim etc/hadoop/slaves
Slave01
Slave02
cd /usr/local/hadoop
bin/hdfs namenode -format
sbin/start-all.sh
教材第86页
cd /usr/local/hadoop
./bin/hdfs dfs -mkdir -p /user/hadoop/input
./bin/hdfs dfs -put ./etc/hadoop/*.xml /user/hadoop/input
./bin/hdfs dfs -ls /user/hadoop/input
教材第87页
./bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar grep /user/hadoop/input output 'dfs[a-z.]+'
./bin/hdfs dfs -cat output/*
|