org.apache.hcatalog.common
Class HCatUtil

Object
  HCatUtil

public class HCatUtil
extends Object

Constructor Summary

HCatUtil()

Method Summary

static boolean	checkJobContextIfRunningFromBackend(JobContext j)
static void	closeHiveClientQuietly(HiveMetaStoreClient client)
static void	configureOutputStorageHandler(HCatStorageHandler storageHandler, Configuration conf, OutputJobInfo outputJobInfo)
static void	copyConf(Configuration src, Configuration dest) Replace the contents of dest with the contents of src
static void	copyJobPropertiesToJobConf(Map<String,String> jobProperties, JobConf jobConf)
static byte[]	decodeBytes(String str)
static Object	deserialize(String str)
static String	encodeBytes(byte[] bytes)
static HCatSchema	extractSchema(Partition partition)
static HCatSchema	extractSchema(Table table)
static Token<? extends AbstractDelegationTokenIdentifier>	extractThriftToken(String tokenStrForm, String tokenSignature)
static Pair<String,String>	getDbAndTableName(String tableName)
static List<FieldSchema>	getFieldSchemaList(List<HCatFieldSchema> hcatFields)
static List<HCatFieldSchema>	getHCatFieldSchemaList(FieldSchema... fields)
static List<HCatFieldSchema>	getHCatFieldSchemaList(List<FieldSchema> fields)
static HiveMetaStoreClient	getHiveClient(HiveConf hiveConf) Get or create a hive client depending on whether it exits in cache or not
static HiveConf	getHiveConf(Configuration conf)
static Map<String,String>	getInputJobProperties(HCatStorageHandler storageHandler, InputJobInfo inputJobInfo)
static JobConf	getJobConfFromContext(JobContext jobContext)
static Token<DelegationTokenIdentifier>	getJobTrackerDelegationToken(Configuration conf, String userName)
static HCatSchema	getPartitionColumns(Table table) return the partition columns from a table instance
static HCatStorageHandler	getStorageHandler(Configuration conf, PartInfo partitionInfo)
static HCatStorageHandler	getStorageHandler(Configuration conf, StorerInfo storerInfo) Create an instance of a storage handler defined in storerInfo.
static HCatStorageHandler	getStorageHandler(Configuration conf, String storageHandler, String serDe, String inputFormat, String outputFormat) Create an instance of a storage handler.
static Table	getTable(HiveMetaStoreClient client, String dbName, String tableName)
static HCatSchema	getTableSchemaWithPtnCols(Table table)
static String	serialize(Serializable obj)
static boolean	validateExecuteBitPresentIfReadOrWrite(FsAction perms) Ensure that read or write permissions are not granted without also granting execute permissions.
static boolean	validateMorePermissive(FsAction first, FsAction second) Test if the first FsAction is more permissive than the second.
static List<FieldSchema>	validatePartitionSchema(Table table, HCatSchema partitionSchema) Validate partition schema, checks if the column types match between the partition and the existing table schema.

Methods inherited from class Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

HCatUtil

public HCatUtil()

Method Detail

checkJobContextIfRunningFromBackend

public static boolean checkJobContextIfRunningFromBackend(JobContext j)

serialize

public static String serialize(Serializable obj)
                        throws IOException

Throws:

IOException

deserialize

public static Object deserialize(String str)
                          throws IOException

Throws:

IOException

encodeBytes

public static String encodeBytes(byte[] bytes)

decodeBytes

public static byte[] decodeBytes(String str)

getHCatFieldSchemaList

public static List<HCatFieldSchema> getHCatFieldSchemaList(FieldSchema... fields)
                                                    throws HCatException

Throws:

HCatException

getHCatFieldSchemaList

public static List<HCatFieldSchema> getHCatFieldSchemaList(List<FieldSchema> fields)
                                                    throws HCatException

Throws:

HCatException

extractSchema

public static HCatSchema extractSchema(Table table)
                                throws HCatException

Throws:

HCatException

extractSchema

public static HCatSchema extractSchema(Partition partition)
                                throws HCatException

Throws:

HCatException

getFieldSchemaList

public static List<FieldSchema> getFieldSchemaList(List<HCatFieldSchema> hcatFields)

getTable

public static Table getTable(HiveMetaStoreClient client,
                             String dbName,
                             String tableName)
                      throws NoSuchObjectException,
                             TException,
                             MetaException

Throws:

NoSuchObjectException

TException

MetaException

getTableSchemaWithPtnCols

public static HCatSchema getTableSchemaWithPtnCols(Table table)
                                            throws IOException

Throws:

IOException

getPartitionColumns

public static HCatSchema getPartitionColumns(Table table)
                                      throws IOException

return the partition columns from a table instance

Parameters:

table - the instance to extract partition columns from

Returns:

HCatSchema instance which contains the partition columns

Throws:

IOException

validatePartitionSchema

public static List<FieldSchema> validatePartitionSchema(Table table,
                                                        HCatSchema partitionSchema)
                                                 throws IOException

Validate partition schema, checks if the column types match between the partition and the existing table schema. Returns the list of columns present in the partition but not in the table.

Parameters:

table - the table

partitionSchema - the partition schema

Returns:

the list of newly added fields

Throws:

IOException - Signals that an I/O exception has occurred.

validateMorePermissive

public static boolean validateMorePermissive(FsAction first,
                                             FsAction second)

Test if the first FsAction is more permissive than the second. This is useful in cases where we want to ensure that a file owner has more permissions than the group they belong to, for eg. More completely(but potentially more cryptically) owner-r >= group-r >= world-r : bitwise and-masked with 0444 => 444 >= 440 >= 400 >= 000 owner-w >= group-w >= world-w : bitwise and-masked with &0222 => 222 >= 220 >= 200 >= 000 owner-x >= group-x >= world-x : bitwise and-masked with &0111 => 111 >= 110 >= 100 >= 000

Returns:

true if first FsAction is more permissive than the second, false if not.

validateExecuteBitPresentIfReadOrWrite

public static boolean validateExecuteBitPresentIfReadOrWrite(FsAction perms)

Ensure that read or write permissions are not granted without also granting execute permissions. Essentially, r-- , rw- and -w- are invalid, r-x, -wx, rwx, ---, --x are valid

Parameters:

perms - The FsAction to verify

Returns:

true if the presence of read or write permission is accompanied by execute permissions

getJobTrackerDelegationToken

public static Token<DelegationTokenIdentifier> getJobTrackerDelegationToken(Configuration conf,
                                                                            String userName)
                                                                     throws Exception

Throws:

Exception

extractThriftToken

public static Token<? extends AbstractDelegationTokenIdentifier> extractThriftToken(String tokenStrForm,
                                                                                    String tokenSignature)
                                                                             throws MetaException,
                                                                                    TException,
                                                                                    IOException

Throws:

MetaException

TException

IOException

getStorageHandler

public static HCatStorageHandler getStorageHandler(Configuration conf,
                                                   StorerInfo storerInfo)
                                            throws IOException

Create an instance of a storage handler defined in storerInfo. If one cannot be found then FosterStorageHandler is used to encapsulate the InputFormat, OutputFormat and SerDe. This StorageHandler assumes the other supplied storage artifacts are for a file-based storage system.

Parameters:

conf - job's configuration will be used to configure the Configurable StorageHandler

storerInfo - StorerInfo to definining the StorageHandler and InputFormat, OutputFormat and SerDe

Returns:

storageHandler instance

Throws:

IOException

getStorageHandler

public static HCatStorageHandler getStorageHandler(Configuration conf,
                                                   PartInfo partitionInfo)
                                            throws IOException

Throws:

IOException

getStorageHandler

public static HCatStorageHandler getStorageHandler(Configuration conf,
                                                   String storageHandler,
                                                   String serDe,
                                                   String inputFormat,
                                                   String outputFormat)
                                            throws IOException

Create an instance of a storage handler. If storageHandler == null, then surrrogate StorageHandler is used to encapsulate the InputFormat, OutputFormat and SerDe. This StorageHandler assumes the other supplied storage artifacts are for a file-based storage system.

Parameters:

conf - job's configuration will be used to configure the Configurable StorageHandler

storageHandler - fully qualified class name of the desired StorageHandle instance

serDe - fully qualified class name of the desired SerDe instance

inputFormat - fully qualified class name of the desired InputFormat instance

outputFormat - fully qualified class name of the desired outputFormat instance

Returns:

storageHandler instance

Throws:

IOException

getDbAndTableName

public static Pair<String,String> getDbAndTableName(String tableName)
                                             throws IOException

Throws:

IOException

getInputJobProperties

public static Map<String,String> getInputJobProperties(HCatStorageHandler storageHandler,
                                                       InputJobInfo inputJobInfo)

configureOutputStorageHandler

@InterfaceAudience.Private
@InterfaceStability.Evolving
public static void configureOutputStorageHandler(HCatStorageHandler storageHandler,
                                                                                                       Configuration conf,
                                                                                                       OutputJobInfo outputJobInfo)

copyConf

public static void copyConf(Configuration src,
                            Configuration dest)

Replace the contents of dest with the contents of src

Parameters:

src -

dest -

getHiveClient

public static HiveMetaStoreClient getHiveClient(HiveConf hiveConf)
                                         throws MetaException,
                                                IOException

Get or create a hive client depending on whether it exits in cache or not

Parameters:

hiveConf - The hive configuration

Returns:

the client

Throws:

MetaException - When HiveMetaStoreClient couldn't be created

IOException

closeHiveClientQuietly

public static void closeHiveClientQuietly(HiveMetaStoreClient client)

getHiveConf

public static HiveConf getHiveConf(Configuration conf)
                            throws IOException

Throws:

IOException

getJobConfFromContext

public static JobConf getJobConfFromContext(JobContext jobContext)

copyJobPropertiesToJobConf

public static void copyJobPropertiesToJobConf(Map<String,String> jobProperties,
                                              JobConf jobConf)