The following procedure installs and configures the LZO compression libraries on Hortonworks Data Platform 2.3 or 2.4.
-
Copy and paste these commands into the shell window on Master node 1:
#!/bin/bash VERSION=$(readlink -e /usr/hdp/current/hadoop-client|cut -d '/' -f4|cut -d '-' -f1) VERSION_UNDERSCORE=$(readlink -e /usr/hdp/current/hadoop-client|cut -d '/' -f4|tr '.' '_'|tr '-' '_') BUILD=$(readlink -e /usr/hdp/current/hadoop-client|cut -d '/' -f4|cut -d '-' -f2) cd /var/opt/teradata/packages/hadoop wget -N https://s3.amazonaws.com/public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/${VERSION}/hadooplzo/hadooplzo_${VERSION_UNDERSCORE}-0.6.0.${VERSION}-${BUILD}.x86_64.rpm wget -N https://s3.amazonaws.com/public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/${VERSION}/hadooplzo/hadooplzo_${VERSION_UNDERSCORE}-native-0.6.0.${VERSION}-${BUILD}.x86_64.rpm wget -N https://s3.amazonaws.com/public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/suse11sp3/hadoop-lzo/hadoop-lzo-0.6.0-1.x86_64.rpm wget -N https://s3.amazonaws.com/public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/suse11sp3/hadoop-lzo/hadoop-lzo-native-0.6.0-1.x86_64.rpm [[ -e hadoop-lzo-0.6.0-1.x86_64.rpm ]] && hcli support install --data --master --edge -r hadoop-lzo-0.6.0-1.x86_64.rpm [[ -e hadoop-lzo-native-0.6.0-1.x86_64.rpm ]] && hcli support install --data --master --edge -r hadoop-lzo-native-0.6.0-1.x86_64.rpm [[ -e hadooplzo_${VERSION_UNDERSCORE}-0.6.0.${VERSION}-${BUILD}.x86_64.rpm ]] && hcli support install --data --master --edge -r hadooplzo_${VERSION_UNDERSCORE}-0.6.0.${VERSION}-${BUILD}.x86_64.rpm [[ -e hadooplzo_${VERSION_UNDERSCORE}-native-0.6.0.2.3.0.0-${BUILD}.x86_64.rpm ]] && hcli support install --data --master --edge -r hadooplzo_${VERSION_UNDERSCORE}-native-0.6.0.2.3.0.0-${BUILD}.x86_64.rpm # Update Ambari, hdfs-site and core-site hcli config set core-site io.compression.codecs 'org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec' hcli config set core-site io.compression.codec.lzo.class 'com.hadoop.compression.lzo.LzoCodec' hcli node runonall "cp /usr/hdp/current/hadoop-client/lib/hadoop-lzo-*.jar /usr/hdp/current/oozie-client/libext/" #Optional, enable mapred compression #hcli config set mapred-site mapred.output.compression.codec 'com.hadoop.compression.lzo.LzopCodec' hcli system restart --stale cd -