珂珂的个人博客 - 一个程序猿的个人网站
功能不是太多。。。根据接口配置可以实现任意的采集
添加文章定义的接口，就是往数据库插入数据
01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 | public interface IGather { FYJ.Data.IDbHelper Db { get; set; } ////// 表名前缀 ///string TablePre { get; set; } ////// 添加 /////////MessageEx Insert(Post model); ////// 获取分类 //////CateDataSource GetCate(); ////// 产生一个新的ID //////string GetNewID(); } |
分析采集源定义的接口,全是属性，主要获取来源，作者，分析内容，我们采集别人的总要给个原来链接和来源吧。
01 02 03 04 05 06 07 08 09 10 11 12 13 | public interface IAnalyze { string Url { get; } string Html { get;} string Title { get; } string Author { get;} string Content { get; } string FromSource { get;} string Keywords { get;} string Description { get;} } |
下面贴上部分源码...有点复杂
001 002 003 004 005 006 007 008 009 010 011 012 013 014 015 016 017 018 019 020 021 022 023 024 025 026 027 028 029 030 031 032 033 034 035 036 037 038 039 040 041 042 043 044 045 046 047 048 049 050 051 052 053 054 055 056 057 058 059 060 061 062 063 064 065 066 067 068 069 070 071 072 073 074 075 076 077 078 079 080 081 082 083 084 085 086 087 088 089 090 091 092 093 094 095 096 097 098 099 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 | using GatherTool.Analyze;using GatherTool.Discuz;using GatherTool.Util;using System;using System.Collections.Generic;using System.ComponentModel;using System.Configuration;using System.Data;using System.Drawing;using System.IO;using System.Linq;using System.Net;using System.Text;using System.Text.RegularExpressions;using System.Threading;using System.Windows.Forms;using System.Xml;namespace GatherTool{ public partial class MainForm : Form { public MainForm() { InitializeComponent(); Control.CheckForIllegalCrossThreadCalls = false; } #region 初始化 protected override void OnLoad(EventArgs e) { base.OnLoad(e); LoadClassConfig(); LoadAtt(); LoadAnalyzeClassConfig(); this.AllowDrop = true; this.toolStripProgressBar1.Visible = false; } private ListdbList; private ListtablePreList; //上传文件后fileID为键名,objectTag为键值 private DictionaryfileDic; //网页html中下载远程图片后保存的fileID private ListfileIDList; private void LoadClassConfig() { dbList = new List(); tablePreList = new List(); Listlist = new List(); XmlDocument doc = new XmlDocument(); doc.Load(Path.Combine(Application.StartupPath, "Gather.xml")); foreach (XmlNode node in doc.GetElementsByTagName("Gather")) { list.Add(node.Attributes["type"].Value); tablePreList.Add(node.Attributes["tablePre"].Value); FYJ.Data.IDbHelper db = FYJ.Data.DbFactory.CreateIDbHelper(node.Attributes["providerName"].Value, node.Attributes["connectionString"].Value); dbList.Add(db); } this.cbConfigName.DataSource = list; } private void LoadAnalyzeClassConfig() { Listlist = new List(); XmlDocument doc = new XmlDocument(); doc.Load(Path.Combine(Application.StartupPath, "Gather.xml")); foreach (XmlNode node in doc.GetElementsByTagName("Analyze")) { list.Add(node.Attributes["type"].Value); } this.cbAnalyze.DataSource = list; } //附件权限列表框 private void LoadAtt() { //允许下载 0 需要回复下载1 禁止下载2 DataTable dt = new DataTable(); dt.Columns.Add("name"); dt.Columns.Add("value"); dt.Rows.Add(new object[] { "允许下载", "0" }); dt.Rows.Add(new object[] { "需要回复下载", "1" }); dt.Rows.Add(new object[] { "禁止下载", "2" }); this.cbAttachmentLimit.DataSource = dt; this.cbAttachmentLimit.DisplayMember = "name"; this.cbAttachmentLimit.ValueMember = "value"; } #endregion private IGather Gather { get { if (cbConfigName.Items.Count > 0) { IGather gather = (IGather)Activator.CreateInstance(Type.GetType(this.cbConfigName.Text)); FYJ.Data.IDbHelper db = this.dbList[this.cbConfigName.SelectedIndex]; gather.Db = db; gather.TablePre = this.tablePreList[this.cbConfigName.SelectedIndex]; return gather; } return null; } } private void cbConfigName_SelectedIndexChanged(object sender, EventArgs e) { if (cbConfigName.Items.Count > 0) { FYJ.Data.IDbHelper db = this.dbList[this.cbConfigName.SelectedIndex]; this.txtConnectionString.Text = db.ConnectionString; } } private void btnTest_Click(object sender, EventArgs e) { if (cbConfigName.Items.Count > 0) { FYJ.Data.IDbHelper db = this.dbList[this.cbConfigName.SelectedIndex]; if (db.TestCanConnectionOpen()) { MessageBox.Show("连接成功"); } else { MessageBox.Show("连接失败"); } } } private void discuz用户ToolStripMenuItem_Click(object sender, EventArgs e) { if (!String.IsNullOrEmpty(this.cbConfigName.Text)) { DisUserForm frm = new DisUserForm(); FYJ.Data.IDbHelper db = this.dbList[this.cbConfigName.SelectedIndex]; frm.Db = db; frm.TablePre = this.tablePreList[this.cbConfigName.SelectedIndex]; frm.Show(); } } private void comboBox_category_DropDown(object sender, EventArgs e) { if (!String.IsNullOrEmpty(this.cbConfigName.Text)) { ThreadPool.QueueUserWorkItem((o) =>{ try { this.toolStripStatusLabel1.Text = "正在获取板块/分类列表..."; CateDataSource data = Gather.GetCate(); this.comboBox_category.DataSource = data.DataSource; this.comboBox_category.DisplayMember = data.DisplayMember; this.comboBox_category.ValueMember = data.ValueMember; this.toolStripStatusLabel1.Text = "就绪"; } catch (Exception ex) { this.toolStripStatusLabel1.Text = ex.Message; } }); } } #region 分析 private void btnAnalyze_Click(object sender, EventArgs e) { if (this.txtFromUrl.Text.Trim() == "") { MessageBox.Show("请输入url"); return; } if (cbAnalyze.Items.Count == 0) { MessageBox.Show("没有分析类"); return; } fileIDList = new List(); IAnalyze ana = (IAnalyze)Activator.CreateInstance(Type.GetType(this.cbAnalyze.Text), this.txtFromUrl.Text.Trim()); this.txtSource.Text = ana.FromSource; this.txtTitle.Text = ana.Title; this.txtFromAuthor.Text = ana.Author; this.richTextBox1.Text = ana.Content; ThreadPool.QueueUserWorkItem((o) =>{ try { string url = this.txtFromUrl.Text.Trim(); string content = this.richTextBox1.Text; #region 下载html中的图片并上传到百度空间 if (this.checkBox_下载图片.Checked) { HttpHelper helper = new HttpHelper(); ListimageList = helper.GetTagAttribute(content, "img", "src");//图片列表 string html = string.Empty; UploadHelper up = new UploadHelper(); this.toolStripProgressBar1.Visible = true; this.toolStripProgressBar1.Maximum = imageList.Count; for (int i = 0; i < imageList.Count; i++) { try { string imageRealPath = helper.GetRealPath(imageList[i], url); string fileName = Path.GetFileName(imageRealPath.IndexOf("?") == -1 ? imageRealPath : imageRealPath.Substring(0, imageRealPath.IndexOf("?"))); this.logList1.AddItem("分析到远程图片:" + imageRealPath); this.toolStripStatusLabel1.Text = ("正在从" + imageRealPath + "下载图片..." + (i + 1) + "/" + imageList.Count); HttpWebRequest req = (HttpWebRequest)WebRequest.Create(imageRealPath); req.Proxy = null; //解决.net 4 第一次请求慢的问题 req.KeepAlive = true; req.Method = "GET"; req.AllowAutoRedirect = true; req.ContentType = "application/x-www-form-urlencoded"; req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; req.Timeout = 10000; HttpWebResponse res = (HttpWebResponse)req.GetResponse(); Stream stream = res.GetResponseStream(); string str = up.UploadToBaidu(stream, Path.GetFileName(imageList[i])); FYJ.Common.JsonHelper json = new FYJ.Common.JsonHelper(str); if (json.GetValue("code") != "1") { this.logList1.AddErrorItem("下载" + imageRealPath + "失败," + str); } if (!fileIDList.Contains(json.GetValue("fileID"))) { fileIDList.Add(json.GetValue("fileID")); } //替换img标签的图片 content = content.Replace(imageList[i], json.GetValue("url")); } catch (Exception ex) { this.logList1.AddErrorItem("下载" + imageList[i] + "失败," + ex.Message); } this.toolStripProgressBar1.Value = i + 1; } this.richTextBox1.Text = content; } #endregion } catch (Exception ex) { this.logList1.AddErrorItem("分析" + this.txtFromUrl.Text.Trim() + "失败," + ex.Message); } this.toolStripStatusLabel1.Text = "就绪"; this.toolStripProgressBar1.Visible = false; }); } #endregion #region 上传附件 private void btnUploadAttachment_Click(object sender, EventArgs e) { if (this.listBoxFile.Items.Count == 0) { MessageBox.Show("没有附件供上传"); return; } fileDic = new Dictionary(); ThreadPool.QueueUserWorkItem((o) =>{ UploadHelper up = new UploadHelper(); this.toolStripProgressBar1.Visible = true; this.toolStripProgressBar1.Maximum = this.listBoxFile.Items.Count; for (int i = 0; i < this.listBoxFile.Items.Count; i++) { string filePath = this.listBoxFile.Items[i].ToString(); try { this.toolStripStatusLabel1.Text = ("正在上传文件..." + (i + 1) + "/" + this.listBoxFile.Items.Count); Stream stream = new FileStream(filePath, FileMode.Open); string str = up.UploadToBaidu(stream, Path.GetFileName(filePath)); FYJ.Common.JsonHelper json = new FYJ.Common.JsonHelper(str); if (json.GetValue("code") != "1") { this.logList1.AddErrorItem("上传文件" + filePath + "失败," + str); } if (!fileDic.ContainsKey(json.GetValue("fileID"))) { fileDic.Add(json.GetValue("fileID"), "attachment"); } } catch (Exception ex) { this.logList1.AddErrorItem("上传文件" + filePath + "失败," + ex.Message); } this.toolStripProgressBar1.Value=i+1; } this.toolStripProgressBar1.Visible = false; this.toolStripStatusLabel1.Text = "就绪"; }); } #endregion #region 发布 private void btnAdd_Click(object sender, EventArgs e) { if (this.txtTitle.Text.Trim() == "") { MessageBox.Show("标题不能为空"); return; } if (this.richTextBox1.Text.Trim() == "") { MessageBox.Show("内容不能为空"); return; } ThreadPool.QueueUserWorkItem((o) =>{ try { this.toolStripStatusLabel1.Text = "正在发布..."; string postID = Gather.GetNewID(); this.toolStripStatusLabel1.Text = "正在保存文件关系..."; UploadHelper up = new UploadHelper(); this.toolStripProgressBar1.Visible = true; this.toolStripProgressBar1.Maximum = 0; if(fileDic!=null) { this.toolStripProgressBar1.Maximum += fileDic.Count; } if (fileIDList != null) { this.toolStripProgressBar1.Maximum += fileIDList.Count; } if(fileDic!=null) { foreach (string fileID in fileDic.Keys) { up.SaveFileRelationDb(fileID, postID, fileDic[fileID]); this.toolStripProgressBar1.Value++; } } if(fileIDList!=null) { foreach (string fileID in fileIDList) { up.SaveFileRelationDb(fileID, postID, ""); this.toolStripProgressBar1.Value++; } } this.toolStripProgressBar1.Visible = false; this.toolStripStatusLabel1.Text = "正在发布..."; GatherTool.Util.Post model = new GatherTool.Util.Post(); model.PostID = postID; model.AuthorID = this.textBox_发帖用户Id.Text; if (this.comboBox_category.SelectedValue != null) { model.CateID = this.comboBox_category.SelectedValue.ToString(); } model.AuthorUserName = this.textBox1_发帖用户名.Text.Trim(); model.Title = this.txtTitle.Text.Trim(); model.Content = this.richTextBox1.Text.Trim(); model.AttachmentLimit = Convert.ToInt32(this.cbAttachmentLimit.SelectedValue); model.FromAuthor = this.txtFromAuthor.Text.Trim(); model.FromSourceUrl = this.txtFromUrl.Text.Trim(); model.FromSource = this.txtSource.Text.Trim(); GatherTool.Util.MessageEx message = Gather.Insert(model); if (message.Code > 0) { this.logList1.AddItem(message.Message); } else { this.logList1.AddErrorItem(message.Message); } } catch (Exception ex) { this.logList1.AddErrorItem(ex.Message); } this.toolStripStatusLabel1.Text = "就绪"; }); } #endregion #region 拖动 private void listBox1_DragEnter(object sender, DragEventArgs e) { e.Effect = DragDropEffects.Move; } private void listBox1_DragDrop(object sender, DragEventArgs e) { if (e.Data.GetDataPresent(DataFormats.FileDrop, false)) { String[] files = (String[])e.Data.GetData(DataFormats.FileDrop); ListtempList = new List(); foreach (String s in files) { if (File.Exists(s)) { return; } //if (Directory.Exists(s)) //{ // foreach (String f in Directory.GetFiles(s)) // { // tempList.Add(f); // } //} } } } #endregion #region 添加删除附件 private void btnFileAdd_Click(object sender, EventArgs e) { using (OpenFileDialog of = new OpenFileDialog()) { of.Multiselect = true; if (of.ShowDialog() == DialogResult.OK) { foreach(string file in of.FileNames) { foreach (object item in this.listBoxFile.Items) { if (item.ToString() == file) { break; } } this.listBoxFile.Items.Add(file); } } } } private void btnFileDelete_Click(object sender, EventArgs e) { this.listBoxFile.Items.Remove(this.listBoxFile.SelectedItem); } #endregion }} |