凌晨接到hive作业异常，hive版本为1.2.1，hadoop版本apache 2.7.1，元数据存储在mysql中，异常信息如下： Logging initialized using configuration in jar:file:/opt/apache-hive-1.2.1-bin/lib/hive-common-1.2.1.jar!/hive-log4j.properties Exception in thread "main" java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState. at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver. at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl. at at org.apache.hadoop.util.RunJar.run(RunJar. at org.apache.hadoop.util.RunJar.main(RunJar. Caused by: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils. at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient. at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient. at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient. at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive. at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive. at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState. ... 8 more Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl. at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl. at at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils. ... 14 more Caused by: MetaException(message:Metastore contains multiple versions (2) [ version = 1.2.0, comment = Set by MetaStore *@*.*.*.* ] [ version = 1.2.0, comment = Set by MetaStore *@*.*.*.* ]) at org.apache.hadoop.hive.metastore.ObjectStore.getMSchemaVersion(ObjectStore. at org.apache.hadoop.hive.metastore.ObjectStore.getMetaStoreSchemaVersion(ObjectStore. at org.apache.hadoop.hive.metastore.ObjectStore.checkSchema(ObjectStore. at org.apache.hadoop.hive.metastore.ObjectStore.verifySchema(ObjectStore. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl. at at org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy. at $Proxy9.verifySchema(Unknown Source) at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore. at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore. at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore. at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler. at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler. at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore. at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient. at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient
异常是因为在启动hive命令时会检查hive源数据中有一张VERSION表，如果元数据版本信息获取不到（原因可能是元数据库异常||网络异常||短期内作业量较多操作都会造成查询不到版本信息），这种情况下会判断hive.metastore.schema.verification属性是true还是false，为true时直接抛出MetaException，为false时打出warn警告然后插入一条version数据（这种情况下会造成多条version记录后面的作业会受影响），下面为hive-metastore包中ObjectStore类代码，标红处为造成多条version记录的代码；
private synchronized void checkSchema() throws MetaException { // recheck if it got verified by another thread while we were waiting if (isSchemaVerified.get()) { return; } boolean strictValidation = HiveConf.getBoolVar(getConf(), HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION); // read the schema version stored in metastore db String schemaVer = getMetaStoreSchemaVersion(); if (schemaVer == null) { if (strictValidation) { throw new MetaException("Version information not found in metastore. "); } else { LOG.warn("Version information not found in metastore. " + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() + " is not enabled so recording the schema version " + MetaStoreSchemaInfo.getHiveSchemaVersion()); setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(), "Set by MetaStore " + USER + "@" + HOSTNAME); } } else { // metastore schema version is different than Hive distribution needs if (schemaVer.equalsIgnoreCase(MetaStoreSchemaInfo.getHiveSchemaVersion())) { LOG.debug("Found expected HMS version of " + schemaVer); } else { if (strictValidation) { throw new MetaException("Hive Schema version " + MetaStoreSchemaInfo.getHiveSchemaVersion() + " does not match metastore's schema version " + schemaVer + " Metastore is not upgraded or corrupt"); } else { LOG.error("Version information found in metastore differs " + schemaVer + " from expected schema version " + MetaStoreSchemaInfo.getHiveSchemaVersion() + ". Schema verififcation is disabled " + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION + " so setting version."); setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(), "Set by MetaStore " + USER + "@" + HOSTNAME); } } } isSchemaVerified.set(true); return; }
解决方案：
hive安装好后将hive-site.xml中hive.metastore.schema.verification设置为true，version获取不到时报出异常，不去插入version信息，这样本作业执行失败不会影响下游作业；
开启metastore服务，hive统一连接metastore，由守护进程启动metastore，避免大量hive脚本初始化元数据信息时获取不到版本信息；
优化hive元数据库；
修改观察几天通过grep "Version information not found in metastore" hive.log发现没有再报找不到version的异常了；