根据url生成唯一的idkey,记录并分享:
public class UrlToUniqueKey { private static Hashtable generators = new Hashtable(10); public static long[,] polynomials = new long[65, 2] { { 0L, 0L }, { -4611686018427387904L, -4611686018427387904L }, { -2305843009213693952L, -2305843009213693952L }, { -3458764513820540928L, -5764607523034234880L }, { -576460752303423488L, -576460752303423488L }, { -1441151880758558720L, -4899916394579099648L }, { -2738188573441261568L, -5332261958806667264L }, { -1945555039024054272L, -1945555039024054272L }, { -7602076171001397248L, -3134505340649865216L }, { -9169328841326329856L, -8628896886041870336L }, { -5719571526760529920L, -1612288666598637568L }, { -2742692173068632064L, -3913628076184961024L }, { -1776670052997660672L, -2407174000829530112L }, { -2005227734086713344L, -7718043861406187520L }, { -3112550292466434048L, -5147051424131055616L }, { -2113595600120315904L, -7045600142044430336L }, { -3682677857793867776L, -7529033414544982016L }, { -2812286861060341760L, -6211097204841512960L }, { -7338580408428134400L, -181868019327172608L }, { -4070603152259284992L, -7580349821236543488L }, { -4690912428278415360L, -1656541810593366016L }, { -8730258663983284224L, -1124153882377715712L }, { -7455702586042089472L, -7995369281233616896L }, { -58297206016311296L, -5584750510474264576L }, { -8198983991290757120L, -6900195776585007104L }, { -3368309616348758016L, -777578196575977472L }, { -7950798515817414656L, -5274165001345564672L }, { -1814110279349305344L, -3073563053254508544L }, { -7333090856108294144L, -299958969764413440L }, { -5862047463258456064L, -1088406921112715264L }, { -2699705949936943104L, -6180435270106611712L }, { -5010893113942605824L, -745326200523587584L }, { -7247248171544346624L, -7637035222663430144L }, { -379617898928275456L, -2128952956840574976L }, { -8982278211127738368L, -8328970742175629312L }, { -2699136824088985600L, -657743201922187264L }, { -5723859659287166976L, -3191693334452109312L }, { -5937852116755283968L, -8585462099679903744L }, { -4340513297460625408L, -2535761738024878080L }, { -7223736098545991680L, -5810321976182439936L }, { -8771886216598519808L, -8198507976365965312L }, { -1991715210309664768L, -3768584310714531840L }, { -9162551332419141632L, -8616797614972600320L }, { -719027294788648960L, -6649233720883544064L }, { -3559483092657242112L, -7475962079365038080L }, { -713657193466691584L, -3981624362359062528L }, { -2043424698022887424L, -3853773324916752384L }, { -4041900042031005696L, -2750302262364798976L }, { -4436928417950760960L, -8365357674703716352L }, { -5903553087924551680L, -232537924545363968L }, { -8728448178608234496L, -4183444001187078144L }, { -2341229233224585216L, -6731777037870903296L }, { -1649801635265304576L, -2988673641163274240L }, { -7620924682850272256L, -4999586984279010304L }, { -6766504171168936448L, -3233736554084598272L }, { -4307058212505990400L, -7352830162319492864L }, { -6559722688844616064L, -3544821563555732608L }, { -5503660899170514624L, -9177641346571454016L }, { -187437317177603296L, -1362648669985435232L }, { -24562796859796592L, -1298422235250900464L }, { -3825548837209328312L, -4007863133653759080L }, { -5023278451590057620L, -8135395730479815484L }, { -1370071926941500434L, -7957612307809224958L }, { -2163743556238479977L, -2346333074831914631L }, { -2935493925047926053L, -7378668602973054136L } }; public static UrlToUniqueKey std64 = UrlToUniqueKey.make(UrlToUniqueKey.polynomials[64, 0], 64); public static UrlToUniqueKey std32 = UrlToUniqueKey.make(UrlToUniqueKey.polynomials[32, 0], 32); public static UrlToUniqueKey std40 = UrlToUniqueKey.make(UrlToUniqueKey.polynomials[40, 0], 40); public static UrlToUniqueKey std24 = UrlToUniqueKey.make(UrlToUniqueKey.polynomials[24, 0], 24); private long zero = 0L; private long one = long.MinValue; public long empty; public int degree; public long polynomial; private long[,] ByteModTable; private UrlToUniqueKey(long polynomial, int degree) { this.degree = degree; this.polynomial = polynomial; this.ByteModTable = new long[16, 256]; long[] numArray = new long[128]; long num1 = long.MinValue; long num2 = this.rightMove(long.MinValue, degree - 1); for (int index = 0; index < 128; ++index) { numArray[index] = num1; bool flag = (num1 & num2) != 0L; num1 = this.rightMove(num1, 1); if (flag) num1 ^= polynomial; } this.empty = numArray[64]; for (int index1 = 0; index1 < 16; ++index1) { for (int index2 = 0; index2 < 256; ++index2) { long num3 = 0L; for (int index3 = 0; index3 < 8; ++index3) { if ((index2 & 1 << index3) != 0) num3 ^= numArray[(int)sbyte.MaxValue - index1 * 8 - index3]; } this.ByteModTable[index1, index2] = num3; } } } public static UrlToUniqueKey make(long polynomial, int degree) { long num = polynomial; UrlToUniqueKey heritrix = (UrlToUniqueKey)UrlToUniqueKey.generators[(object)1]; if (heritrix == null) { heritrix = new UrlToUniqueKey(polynomial, degree); UrlToUniqueKey.generators.Add((object)num, (object)heritrix); } return heritrix; } private long rightMove(long value, int pos) { if (pos != 0) { long num = long.MaxValue; value >>= 1; value &= num; value >>= pos - 1; } return value; } public static long long_rightMove(long value, int pos) { if (pos != 0) { long num = long.MaxValue; value >>= 1; value &= num; value >>= pos - 1; } return value; } private int int_rightMove(int value, int pos) { if (pos != 0) { int num = int.MaxValue; value >>= 1; value &= num; value >>= pos - 1; } return value; } public long reduce(long fp) { int num1 = 8 - this.degree / 8; long num2 = num1 == 8 ? 0L : fp & -1L << 8 * num1; long num3 = 0L; for (int index = 0; index < num1; ++index) { num3 ^= this.ByteModTable[8 + index, (int)fp & (int)byte.MaxValue]; fp = this.rightMove(fp, 8); } return num2 ^ num3; } public long extend_byte(long f, int v) { f ^= (long)((int)byte.MaxValue & v); int num = (int)f; return this.rightMove(f, 8) ^ this.ByteModTable[7, num & (int)byte.MaxValue]; } public long extend_char(long f, int v) { f ^= (long)((int)ushort.MaxValue & v); int num = (int)f; return this.rightMove(f, 16) ^ this.ByteModTable[6, num & (int)byte.MaxValue] ^ this.ByteModTable[7, this.int_rightMove(num, 8) & (int)byte.MaxValue]; } public long extend_int(long f, int v) { f ^= (long)uint.MaxValue & (long)v; int num1 = (int)f; long num2 = this.rightMove(f, 32) ^ this.ByteModTable[4, num1 & (int)byte.MaxValue]; int num3 = this.int_rightMove(num1, 8); long num4 = num2 ^ this.ByteModTable[5, num3 & (int)byte.MaxValue]; int num5 = this.int_rightMove(num3, 8); return num4 ^ this.ByteModTable[6, num5 & (int)byte.MaxValue] ^ this.ByteModTable[7, this.int_rightMove(num5, 8) & (int)byte.MaxValue]; } public long extend_long(long f, long v) { f ^= v; long num1 = this.ByteModTable[0, (int)(f & (long)byte.MaxValue)]; f = this.rightMove(f, 8); long num2 = num1 ^ this.ByteModTable[1, (int)(f & (long)byte.MaxValue)]; f = this.rightMove(f, 8); long num3 = num2 ^ this.ByteModTable[2, (int)(f & (long)byte.MaxValue)]; f = this.rightMove(f, 8); long num4 = num3 ^ this.ByteModTable[3, (int)(f & (long)byte.MaxValue)]; f = this.rightMove(f, 8); long num5 = num4 ^ this.ByteModTable[4, (int)(f & (long)byte.MaxValue)]; f = this.rightMove(f, 8); long num6 = num5 ^ this.ByteModTable[5, (int)(f & (long)byte.MaxValue)]; f = this.rightMove(f, 8); long num7 = num6 ^ this.ByteModTable[6, (int)(f & (long)byte.MaxValue)]; f = this.rightMove(f, 8); return num7 ^ this.ByteModTable[7, (int)(f & (long)byte.MaxValue)]; } public long fp(byte[] buf, int start, int n) { return this.extend(this.empty, buf, start, n); } public long fp(char[] buf, int start, int n) { return this.extend(this.empty, buf, start, n); } public long fp(string s) { return this.extend(this.empty, s); } public long fp(int[] buf, int start, int n) { return this.extend(this.empty, buf, start, n); } public long fp(long[] buf, int start, int n) { return this.extend(this.empty, buf, start, n); } public long fp8(string s) { return this.extend8(this.empty, s); } public long fp8(char[] buf, int start, int n) { return this.extend8(this.empty, buf, start, n); } public long extend(long f, byte v) { return this.reduce(this.extend_byte(f, (int)v)); } public long extend(long f, char v) { return this.reduce(this.extend_char(f, (int)v)); } public long extend(long f, int v) { return this.reduce(this.extend_int(f, v)); } public long extend(long f, long v) { return this.reduce(this.extend_long(f, v)); } public long extend(long f, byte[] buf, int start, int n) { for (int index = 0; index < n; ++index) f = this.extend_byte(f, (int)buf[start + index]); return this.reduce(f); } public long extend(long f, char[] buf, int start, int n) { for (int index = 0; index < n; ++index) f = this.extend_char(f, (int)buf[start + index]); return this.reduce(f); } public long extend(long f, string s) { int length = s.Length; for (int index = 0; index < length; ++index) { int v = (int)s[index]; f = this.extend_char(f, v); } return this.reduce(f); } public long extend(long f, int[] buf, int start, int n) { for (int index = 0; index < n; ++index) f = this.extend_int(f, buf[start + index]); return this.reduce(f); } public long extend(long f, long[] buf, int start, int n) { for (int index = 0; index < n; ++index) f = this.extend_long(f, buf[start + index]); return this.reduce(f); } public long extend8(long f, string s) { int length = s.Length; for (int index = 0; index < length; ++index) { int v = (int)s[index]; f = this.extend_byte(f, v); } return this.reduce(f); } public long extend8(long f, char[] buf, int start, int n) { for (int index = 0; index < n; ++index) f = this.extend_byte(f, (int)buf[start + index]); return this.reduce(f); } public static ulong createUrlkey(string url) { int length = url.IndexOf("://"); if (length > 0) length = url.IndexOf('/', length + "://".Length); string s = length == -1 ? url : url.Substring(0, length); return Convert.ToUInt64(Convert.ToString(UrlToUniqueKey.std32.fp(s) | UrlToUniqueKey.long_rightMove(UrlToUniqueKey.std40.fp(url), 24), 2).PadLeft(64, '0'), 2); } /// <summary> /// 根据url生成唯一id /// </summary> /// <param name="url"></param> /// <returns></returns> public static long createUrlToLong(string url) { int length = url.IndexOf("://"); if (length > 0) length = url.IndexOf('/', length + "://".Length); string s = length == -1 ? url : url.Substring(0, length); return UrlToUniqueKey.std32.fp(s) | UrlToUniqueKey.long_rightMove(UrlToUniqueKey.std40.fp(url), 24); } }