【原创】Scrapyd 的 .net 客户端

最近项目需要部署Scrapy爬虫，采用最简单的Scrapyd服务进行部署，基于.net core 进行了客户端的封装。

1）Scrapyd API文档：http://scrapyd.readthedocs.io/en/latest/api.html

2）使用HttpClient进行交互

比较麻烦的是部署爬虫所需要的使用 HttpClient 上传文件的代码，使用了 “MultipartFormDataContent” 对象进行上传，添加字符串内容，及StreamContent，直接可以将MVC中的File对象传入，不用自行处理临时文件。

 var content = new MultipartFormDataContent("Upload-----" + Guid.NewGuid().ToString().Replace("-", ""));

                content.Add(new StringContent(project), ""project"");
                content.Add(new StringContent(version), ""version"");
                content.Add(new StreamContent(eggStream), ""egg"", """ + project + ".egg" + """);

                var resp = client.PostAsync("addversion.json", content).Result;

                var strResult = resp.Content.ReadAsStringAsync().Result;

                return JsonConvert.DeserializeObject<AddVersionResponse>(strResult);

完整代码

  1 using System;
  2 using System.Collections.Generic;
  3 using System.IO;
  4 using System.Linq;
  5 using System.Net.Http;
  6 using System.Text;
  7 using Newtonsoft.Json;
  8 using Scrapyd.Client.Response;
  9 
 10 namespace Scrapyd.Client
 11 {
 12     /// <summary>
 13     /// A Client used to communicate with the Scrapyd Daemon
 14     /// </summary>
 15     public class ScrapydClient
 16     {
 17         private readonly string _host;
 18         private readonly int _port;
 19 
 20         public ScrapydClient(string host, int port)
 21         {
 22             _host = host;
 23             _port = port;
 24         }
 25 
 26         /// <summary>
 27         /// 查询状态
 28         /// </summary>
 29         /// <returns></returns>
 30         public DaemonStatusResponse QueryStatus()
 31         {
 32             HttpClient client = new HttpClient()
 33             {
 34                 BaseAddress = new Uri($"http://{_host}:{_port}")
 35             };
 36 
 37             try
 38             {
 39                 var strResult = client.GetStringAsync("daemonstatus.json").Result;
 40 
 41                 var jresult = JsonConvert.DeserializeObject<DaemonStatusResponse>(strResult);
 42                 return jresult;
 43             }
 44             catch (Exception e)
 45             {
 46                 return new DaemonStatusResponse { status = "failed", message = e.Message };
 47             }
 48         }
 49 
 50         public AddVersionResponse AddVersion(string project, string version, Stream eggStream)
 51         {
 52             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
 53             if (eggStream == null) throw new ArgumentNullException(nameof(eggStream));
 54 
 55             var client = new HttpClient()
 56             {
 57                 BaseAddress = new Uri($"http://{_host}:{_port}")
 58             };
 59 
 60             try
 61             {
 62                 var content = new MultipartFormDataContent("Upload-----" + Guid.NewGuid().ToString().Replace("-", ""));
 63 
 64                 content.Add(new StringContent(project), ""project"");
 65                 content.Add(new StringContent(version), ""version"");
 66                 content.Add(new StreamContent(eggStream), ""egg"", """ + project + ".egg" + """);
 67 
 68                 var resp = client.PostAsync("addversion.json", content).Result;
 69 
 70                 var strResult = resp.Content.ReadAsStringAsync().Result;
 71 
 72                 return JsonConvert.DeserializeObject<AddVersionResponse>(strResult);
 73             }
 74             catch (Exception e)
 75             {
 76                 return new AddVersionResponse { status = "failed", message = e.Message };
 77             }
 78         }
 79 
 80         public ScheduleResponse ScheduleJob(string project, string spider,
 81             string setting = "",
 82             string jobid = "",
 83             string version = "",
 84             Dictionary<string, string> arguments = null)
 85         {
 86             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
 87             if (string.IsNullOrEmpty(spider)) throw new ArgumentNullException(nameof(spider));
 88 
 89 
 90             var client = new HttpClient()
 91             {
 92                 BaseAddress = new Uri($"http://{_host}:{_port}")
 93             };
 94 
 95             try
 96             {
 97                 var data = new Dictionary<string, string>()
 98                 {
 99                     {"project", project},
100                     {"spider", spider}
101                 };
102 
103                 if (!string.IsNullOrEmpty(jobid))
104                 {
105                     data.Add("jobid", jobid);
106                 }
107                 if (!string.IsNullOrEmpty(setting))
108                 {
109                     data.Add("setting", setting);
110                 }
111 
112                 if (!string.IsNullOrEmpty(version))
113                 {
114                     data.Add("_version", version);
115                 }
116 
117                 if (arguments != null && arguments.Count > 0)
118                 {
119                     foreach (var argument in arguments)
120                     {
121                         data.Add(argument.Key, argument.Value);
122                     }
123                 }
124 
125                 var strResult =
126                     client.PostAsync("schedule.json", new FormUrlEncodedContent(data))
127                         .Result.Content.ReadAsStringAsync()
128                         .Result;
129 
130                 return JsonConvert.DeserializeObject<ScheduleResponse>(strResult);
131 
132             }
133             catch (Exception e)
134             {
135                 return new ScheduleResponse { status = "failed", message = e.Message };
136             }
137         }
138 
139         public CancelResponse CancelJob(string project, string jobid)
140         {
141             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
142             if (string.IsNullOrEmpty(jobid)) throw new ArgumentNullException(nameof(jobid));
143 
144 
145             var client = new HttpClient()
146             {
147                 BaseAddress = new Uri($"http://{_host}:{_port}")
148             };
149 
150             try
151             {
152                 var content = new FormUrlEncodedContent(new Dictionary<string, string>()
153                 {
154                     {"project", project},
155                     {"job", jobid}
156                 });
157                 var strResult = client.PostAsync($"/cancel.json", content).Result.Content.ReadAsStringAsync().Result;
158 
159                 return JsonConvert.DeserializeObject<CancelResponse>(strResult);
160 
161             }
162             catch (Exception e)
163             {
164                 return new CancelResponse { status = "failed", message = e.Message };
165             }
166         }
167 
168         public ListProjectsResponse ListProjects()
169         {
170             try
171             {
172                 var client = new HttpClient
173                 {
174                     BaseAddress = new Uri($"http://{_host}:{_port}")
175                 };
176                 var result = client.GetStringAsync("listprojects.json").Result;
177                 return JsonConvert.DeserializeObject<ListProjectsResponse>(result);
178             }
179             catch (Exception e)
180             {
181                 return new ListProjectsResponse { status = "failed", message = e.Message };
182             }
183         }
184 
185         public ListVersionsResponse ListVersions(string project)
186         {
187             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
188 
189             try
190             {
191                 var client = new HttpClient
192                 {
193                     BaseAddress = new Uri($"http://{_host}:{_port}")
194                 };
195 
196                 var result = client.GetStringAsync($"listversions.json?project={project}").Result;
197                 return JsonConvert.DeserializeObject<ListVersionsResponse>(result);
198             }
199             catch (Exception e)
200             {
201                 return new ListVersionsResponse { status = "failed", message = e.Message };
202             }
203         }
204 
205         public ListSpidersResponse ListSpiders(string project, string version = "")
206         {
207             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
208 
209             try
210             {
211                 var client = new HttpClient
212                 {
213                     BaseAddress = new Uri($"http://{_host}:{_port}")
214                 };
215 
216                 var result = client.GetStringAsync($"listspiders.json?project={project}&_version={version}").Result;
217                 return JsonConvert.DeserializeObject<ListSpidersResponse>(result);
218             }
219             catch (Exception e)
220             {
221                 return new ListSpidersResponse { status = "failed", message = e.Message };
222             }
223         }
224 
225         public ListJobsResponse ListJobs(string project)
226         {
227             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
228 
229             try
230             {
231                 var client = new HttpClient
232                 {
233                     BaseAddress = new Uri($"http://{_host}:{_port}")
234                 };
235 
236                 var result = client.GetStringAsync($"listjobs.json?project={project}").Result;
237                 return JsonConvert.DeserializeObject<ListJobsResponse>(result);
238             }
239             catch (Exception e)
240             {
241                 return new ListJobsResponse { status = "failed", message = e.Message };
242             }
243         }
244 
245         public ScrapydResponse DeleteVersion(string project, string version)
246         {
247             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
248             if (string.IsNullOrEmpty(version)) throw new ArgumentNullException(nameof(version));
249 
250             var client = new HttpClient()
251             {
252                 BaseAddress = new Uri($"http://{_host}:{_port}")
253             };
254 
255             try
256             {
257                 var data = new Dictionary<string, string>()
258                 {
259                     {"project", project},
260                     {"version", version}
261                 };
262 
263                 var strResult =
264                     client.PostAsync("delversion.json", new FormUrlEncodedContent(data))
265                         .Result.Content.ReadAsStringAsync()
266                         .Result;
267 
268                 return JsonConvert.DeserializeObject<ScrapydResponse>(strResult);
269 
270             }
271             catch (Exception e)
272             {
273                 return new ScrapydResponse { status = "failed", message = e.Message };
274             }
275         }
276 
277         public ScrapydResponse DeleteProject(string project)
278         {
279             if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
280 
281 
282             var client = new HttpClient()
283             {
284                 BaseAddress = new Uri($"http://{_host}:{_port}")
285             };
286 
287             try
288             {
289                 var data = new Dictionary<string, string>()
290                 {
291                     {"project", project}
292                 };
293 
294                 var strResult =
295                     client.PostAsync("delproject.json", new FormUrlEncodedContent(data))
296                         .Result.Content.ReadAsStringAsync()
297                         .Result;
298 
299                 return JsonConvert.DeserializeObject<ScrapydResponse>(strResult);
300 
301             }
302             catch (Exception e)
303             {
304                 return new ScrapydResponse { status = "failed", message = e.Message };
305             }
306         }
307 
308     }
309 
310 }

博客地址 : http://www.cnblogs.com/mobwiz/