帝国后台导入新闻
将采集到的新闻导入到数据库中
<?php
include_once ('MysqlDB.class.php');
include_once("getSource.php");
$db = MysqlDB::getInstance('127.0.0.1', 'multilang.sbmchina.com', 'root','');
//$data = $db->fetch("phome_ecms_news_index"," WHERE id =655");
//$data=$query->fetchAll();
//print_r($data);
$urls=getUrls("urls/ar_url.log");
//
$filename=298;
foreach ($urls as $url) {
$source=getSource($url);
$title = $source['title'];
$time = $source['time'];
$content = $source['content'];
//产品数据表中字段
$newsfield = array(
"classid" => 56,
"ttid" => 0,
"onclick" => 0,
"plnum" => 0,
"totaldown" => 0,
"newspath" => "",
"filename" => $filename,
"userid" => 1,
"username" =>"超管",
"firsttitle" => 0,
"isgood" => 0,
"ispic" => 0,
"istop" => 0,
"isqf" => 0,
"ismember" => 0,
"isurl" => 0,
"truetime" => $time,
"lastdotime" => $time,
"havehtml" => 1,
"groupid" => 0,
"userfen" => 0,
"titlefont" => "",
"titleurl" => "http://fr.sbmchina.com/media/news/{$filename}.html",
"stb" => 1,
"fstb" => 1,
"restb" => 1,
"keyboard" => "",
"title" => $title,
"newstime" => $time,
"titlepic" => "",
"ftitle" => "",
"smalltext" => $title,
"diggtop" => 0,
"relatedmachine" => "",
"relatedinfo" => "",
"classname"=>"",
"tags"=>""
);
$data = $db->insert("phome_ecms_news", $newsfield);
echo "insert phome_ecms_news success".PHP_EOL;
$re = $db->query("select * from phome_ecms_news where title='{$title}'");
$result=$re->fetch();
$id=$result['id'];
$classid=$result['classid'];
//产品数据分表中字段
$proDataField = array(
"id" => $id,
"classid" => $classid,
"keyid" =>"",
"dokey" => 1,
"newstempid" => 0,
"closepl" => 0,
"haveaddfen" => 0,
"infotags" =>"" ,
"writer" =>"",
"befrom"=>"",
"newstext"=>$content
);
$data = $db->insert("phome_ecms_news_data_1", $proDataField);
echo "insert phome_ecms_news_data_1 success".PHP_EOL;
//产品索引表中字段
$proIndex = array(
"id" => $id,
"classid" => $classid,
"checked" => 1,
"newstime" => $time,
"truetime" => $time,
"lastdotime" => $time,
"havehtml" => 1
);
//print_r($proIndex);
$data = $db->insert("phome_ecms_news_index", $proIndex);
echo 'insert into phome_ecms_news_index success'.PHP_EOL;
$filename++;
}
$db->close();
?>根据url抓取内容
<?php
@header("Content-Type: text/html; charset=UTF-8");
//不限执行时间
set_time_limit(0);
error_reporting(0);
//从日志文件中获取url
function getUrls($fileName){
$lineArray=array();
$fp = fopen($fileName, "rb") or die("Unable to open file ".$fileName);
while(! feof($fp))
{
$line=fgets($fp);
$line=trim($line);
$lineArr[]=$line;
}
fclose($fp);
return $lineArr;
}
//根据url获取内容
function getSource($url){
$title="";
$time="";
$content="";
require_once('phpQuery/phpQuery.php');
$html=phpQuery::newDocumentFile($url);
//获取描述内容
$title=$html->find('.newstitle')->html();
$time=$html->find('.newstime')->html();
$time=strtotime($time);
$content=$html->find('.newscontent')->html();
$search = array(
"~<span(.*?)>~is",
"~<p (.*?)>~is"
);
$replace = array(
"<span>",
"<p>"
);
//去除样式，去除后效果不是太好不用处理
//$content = preg_replace($search,$replace,$content);
$content=mb_convert_encoding($content, 'utf-8',mb_detect_encoding($content));
$title=mb_convert_encoding($title, 'utf-8',mb_detect_encoding($title));
//$title=htmlspecialchars($title);
$title= addslashes($title);
//$title=mysql_escape_string ($title);
$content=addslashes($content);
//$content=mysql_escape_string ($content);
//$title=clear($title);
return array('title'=>$title,'time'=>$time,'content'=>$content);
}
function clear($kw)
{
$kw= str_replace(array("~" ,"!" ,"@" ,"#" ,"$" ,"%" ,"^" ,"+","&" ,"*" ,"," ,"." ,"?" ,";",":" ,'\'','"' ,"[" ,"]" ,"{" ,"}" ,"!" ,"￥" ,"……" ,"…" ,"、" ,"，" ,"。" ,"？" ,"；" ,"：","'","“" ,"”" ,"'" ,"【" ,"】" ,"～" ,"！" ,"＠" ,"＃" ,"＄" ,"％" ,"＾" ,"＆" ,"＊" ,"，" ,"．" ,"＜" ,"＞" ,"；" ,"：","＇","＂" ,"［" ,"］" ,"｛" ,"｝","／" ,"＼" ,"（" ,"）" ,"(" ,")","《","》", '$','¿','×'),'', $kw );
$kw= str_replace( array(" ","-","_","\\","/","|"),' ', $kw );
//$kw= str_replace(array("á","í","é","ó","ú","ñ","Á","Í","É","Ó","Ú","Ñ","ç","ã","à","â","ê","ô","õ","ü"),array("a","i","e","o","u","n","a","i","e","o","u","n","c","a","a","a","e","o","o","u"),$kw);
//$kw= str_replace(array("а","б","в","г","д","е","ё","ж","з","и","й","к","л","м","н","о","п","р","с","т","у","ф","х","ц","ч","ш","я","ю","щ","щ","э","ъ","ь","А","Б","В","Г","Д","Е","Ё","Э","Ж","З","И","Й","К","Л","М","Н","О","П","Р","С","Т","У","Ф","Х","Ц","Ч","Ш","Щ","Ы","Ю","Я","ы"),array("a","b","v","g","d","e","e","zh","z","i","j","k","l","m","n","o","p","r","s","t","u","f","x","c","ch","s","ya","yu","sch","y","e","","","A","B","V","G","D","E","E","E","J","Z","I","I","K","L","M","N","O","P","R","S","T","U","F","H","C","CH","SH","SH","Y","YU","YA","s"),$kw);
//$kw = strtolower(strip_tags(trim($kw)));
//$kw = explode(' ',$kw);
//$kw = implode('-',array_filter($kw));
return $kw;
}
// foreach ($urls as $url) {
// $content=getSource($url);
// echo $url.PHP_EOL;
// print_r($content);
// }
//$content=getSource("http://fr.leevii.com/news/204.html");
//print_r($content);
?>数据库操作函数
<?php
class MysqlDB
{
protected static $_instance = null;
protected $dsn;
protected $pdo;
private function __construct($dbHost, $dbName,$dbUser, $dbPasswd, $dbCharset)
{
try {
$this->dsn = 'mysql:host='.$dbHost.';dbname='.$dbName;
$this->pdo = new PDO($this->dsn, $dbUser, $dbPasswd);
$this->pdo->exec('SET character_set_connection='.$dbCharset.', character_set_results='.$dbCharset.', character_set_client=binary');
} catch (PDOException $e) {
echo '数据库连接失败：'.$e->getMessage();
exit;
}
}
//防止PDO类被克隆
private function __clone() {}
public static function getInstance($dbHost='127.0.0.1', $dbName='test',$dbUser='root', $dbPasswd='', $dbCharset='utf8')
{
if (self::$_instance === null) {
self::$_instance = new self($dbHost, $dbName,$dbUser, $dbPasswd, $dbCharset);
}
return self::$_instance;
}
public function fetch($table, $where="",$queryMode = 'All')
{
if(!empty($where)){
$strSql='select * from '.$table.' '.$where;
}else{
$strSql='select * from '.$table;
}
$recordset = $this->pdo->query( $strSql);
if ($recordset) {
$recordset->setFetchMode(PDO::FETCH_ASSOC);
if ($queryMode == 'All') {
$result = $recordset->fetchAll();
} elseif ($queryMode == 'Row') {
$result = $recordset->fetch();
}
} else {
$result = null;
}
return $result;
}
public function update($table, $arrayDataValue, $where = '')
{
if ($where) {
$strSql = '';
foreach ($arrayDataValue as $key => $value) {
$strSql .= ", '$key'='$value'";
}
$strSql = substr($strSql, 1);
$strSql = "UPDATE $table SET $strSql WHERE $where";
} else {
$strSql = "REPLACE INTO $table (".implode(',', array_keys($arrayDataValue)).") VALUES ('".implode("','", $arrayDataValue)."')";
}
$result = $this->pdo->exec($strSql);
return $result;
}
public function insert($table, $arrayDataValue)
{
$strSql = "INSERT INTO $table (".implode(',', array_keys($arrayDataValue)).") VALUES ('".implode("','", $arrayDataValue)."')";
$result = $this->pdo->exec($strSql);
return $result;
}
public function delete($table, $where = '')
{
if ($where == '') {
echo "'WHERE' is Null";
} else {
$strSql = "DELETE FROM $table WHERE $where";
$result = $this->pdo->exec($strSql);
return $result;
}
}
public function exec($strSql)
{
$result = $this->pdo->exec($strSql);
return $result;
}
public function query($strSql)
{
$result = $this->pdo->query($strSql);
return $result;
}
public function close()
{
$this->pdo = null;
}
}
?>
