The following procedure installs and configures the LZO compression libraries on Hortonworks Data Platform 2.5.
-
Copy and paste these commands into the shell window on Master node 1:
#!/bin/bash VERSION=$(readlink -e /usr/hdp/current/hadoop-client|cut -d '/' -f4|cut -d '-' -f1) VERSION_UNDERSCORE=$(readlink -e /usr/hdp/current/hadoop-client \ |cut -d '/' -f4|tr '.' '_'|tr '-' '_') BUILD=$(readlink -e /usr/hdp/current/hadoop-client \ |cut -d '/' -f4|cut -d '-' -f2) cd /var/opt/teradata/packages/hadoop wget -N https://s3.amazonaws.com/public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/\ updates/${VERSION}/hadooplzo/hadooplzo_${VERSION_UNDERSCORE}-0.6.0.${VERSION}-\ ${BUILD}.x86_64.rpm || wget -N https://s3.amazonaws.com\ /private-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/${VERSION}-${BUILD}\ /hadooplzo/hadooplzo_${VERSION_UNDERSCORE}-0.6.0.${VERSION}-${BUILD}.x86_64.rpm wget -N https://s3.amazonaws.com/public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/\ updates/${VERSION}/hadooplzo/hadooplzo_${VERSION_UNDERSCORE}-native-0.6.0.\ ${VERSION}-${BUILD}.x86_64.rpm || wget -N https://s3.amazonaws.com/private-repo-\ 1.hortonworks.com/HDP/suse11sp3/2.x/updates/${VERSION}-${BUILD}\/hadooplzo\ /hadooplzo_${VERSION_UNDERSCORE}-native-0.6.0.${VERSION}-${BUILD}.x86_64.rpm wget -N https://s3.amazonaws.com/public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21\ /repos/suse11sp3/hadoop-lzo/hadoop-lzo-0.6.0-1.x86_64.rpm wget -N https://s3.amazonaws.com/public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21\ /repos/suse11sp3/hadoop-lzo/hadoop-lzo-native-0.6.0-1.x86_64.rpm wget -N https://s3.amazonaws.com/public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21\ /repos/suse11sp3/lzo/lzo-2.06-1.x86_64.rpm wget -N https://s3.amazonaws.com/public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21\ /repos/suse11sp3/lzo/lzo-devel-2.06-1.x86_64.rpm [[ -e lzo-2.06-1.x86_64.rpm ]] && hcli support install --data --master --edge \ -r lzo-2.06-1.x86_64.rpm [[ -e lzo-devel-2.06-1.x86_64.rpm ]] && hcli support install --data --master \ --edge -r lzo-devel-2.06-1.x86_64.rpm [[ -e hadoop-lzo-0.6.0-1.x86_64.rpm ]] && hcli support install --data --master \ --edge -r hadoop-lzo-0.6.0-1.x86_64.rpm [[ -e hadoop-lzo-native-0.6.0-1.x86_64.rpm ]] && hcli support install --data \ --master --edge -r hadoop-lzo-native-0.6.0-1.x86_64.rpm [[ -e hadooplzo_${VERSION_UNDERSCORE}-0.6.0.${VERSION}-${BUILD}.x86_64.rpm ]] \ && hcli support install --data --master --edge -r hadooplzo_${VERSION_UNDERSCORE}\ -0.6.0.${VERSION}-${BUILD}.x86_64.rpm [[ -e hadooplzo_${VERSION_UNDERSCORE}-native-0.6.0.${VERSION}-${BUILD}.x86_64.rpm ]] \ && hcli support install --data --master --edge -r hadooplzo_${VERSION_UNDERSCORE}-\ native-0.6.0.${VERSION}-${BUILD}.x86_64.rpm # Update Ambari, hdfs-site and core-site hcli config set core-site io.compression.codecs 'org.apache.hadoop.io.compress.\ GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.\ compress.SnappyCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.\ lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec' hcli config set core-site io.compression.codec.lzo.class 'com.hadoop.\ compression.lzo.LzoCodec' hcli node runonall "cp /usr/hdp/current/hadoop-client/lib/hadoop-lzo-*.jar \ /usr/hdp/current/oozie-client/libext/" #Optional, enable mapred compression #hcli config set mapred-site mapred.output.compression.codec \ #'com.hadoop.compression.lzo.LzopCodec' hcli system restart --stale cd -