整理列表,生成最终列表

pull/6/head
gently 5 years ago
parent cb23859625
commit b7041c5fc6

3
.gitignore vendored

@ -2,4 +2,5 @@
optimize-dns.conf
.gitignore
.idea/
origin-files/split-tmp-list_*
origin-files/split-tmp-list_*
origin-files/*.php

@ -0,0 +1,424 @@
#TIME=20191214111257
#URL=https://github.com/gentlyxu/anti-AD
address=/uchaoqiang.top/
address=/0082tv.net/
address=/008844.cc/
address=/02123s.com/
address=/023hysj.com/
address=/025suyu.com/
address=/04zl.cn/
address=/0756j.com/
address=/0796gou.com/
address=/100fenlm.com/
address=/114lm.com/
address=/114yeah.com/
address=/116b.com/
address=/1188.com/
address=/12291.com/
address=/123hala.com/
address=/138lm.com/
address=/139site.com/
address=/176um.com/
address=/1kmb.cn/
address=/1x3x.com/
address=/217wo.com/
address=/235123.net/
address=/2898.com/
address=/2f.com/
address=/31365y.com/
address=/321tui.cn/
address=/321tui.com/
address=/3324.cn/
address=/333dm.com/
address=/7clink.com/
address=/88shu.cn/
address=/cnzz.net/
address=/adpush.cn/
address=/adsage.com/
address=/behe.com/
address=/icast.cn/
address=/miaozhen.com/
address=/openxt.cn/
address=/2345.com/
address=/hexun.com/
address=/dopa.com/
address=/dftoutiao.com/
address=/bsmakeup.com.cn/
address=/chinaqirun.cn/
address=/hihufu.cn/
address=/3393.com/
address=/778669.com/
address=/adk2x.com/
address=/adnxs.com/
address=/adsame.com/
address=/alimama.com/
address=/allyes.com/
address=/amazon-adsystem.com/
address=/fastclick.net/
address=/doubleclick.net/
address=/rubiconproject.com/
address=/smartadserver.com/
address=/zedo.com/
address=/admob.com/
address=/inmobi.com/
address=/advertising.com/
address=/51.la/
address=/cnzz.com/
address=/wrating.com/
address=/oadz.com/
address=/allyes.com.cn/
address=/admaster.com.cn/
address=/ok365.com/
address=/youmi.net/
address=/adwo.com/
address=/alimama.cn/
address=/appjiagu.com/
address=/mmstat.com/
address=/linezing.com/
address=/atdmt.com/
address=/adview.cn/
address=/flurry.com/
address=/adfuture.cn/
address=/cooguo.com/
address=/domob.cn/
address=/adsmogo.com/
address=/analysys.cn/
address=/umengcloud.com/
address=/umeng.co/
address=/umeng.com/
address=/appsflyer.com/
address=/authedmine.com/
address=/coin-hive.com/
address=/coinhive.com/
address=/igexin.com/
address=/tanx.com/
address=/inmobi.cn/
address=/51yes.com/
address=/crsspxl.com/
address=/imrworldwide.com/
address=/yiqifa.com/
address=/mob.com/
address=/wooboo.com.cn/
address=/ahksqc.com.cn/
address=/chengjie168.com.cn/
address=/jyzmsy.com.cn/
address=/chuanmen.mobi/
address=/qianyilamian.cn/
address=/jiaxinkang.cn/
address=/55.la/
address=/baifendian.com/
address=/banmamedia.com/
address=/cnzz.cn/
address=/aliapp.org/
address=/ddns.name/
address=/allyes.cn/
address=/dnset.com/
address=/tourstogo.us/
address=/barginginfrance.net/
address=/butlerelectricsupply.com/
address=/cruisingsmallship.com/
address=/frost-electric-supply.com/
address=/iptvdeals.com/
address=/011211.cn/
address=/013572.cn/
address=/020wujin.cn/
address=/0512pifa.com.cn/
address=/0591jiajiao.com.cn/
address=/1357902.cn/
address=/1haows.cn/
address=/4008813318.com.cn/
address=/431.red/
address=/43gw.cn/
address=/467.red/
address=/51juejinjie.com.cn/
address=/555vps.cn/
address=/58xiao.cn/
address=/77av.cn/
address=/77tianxu.cn/
address=/77vip.wang/
address=/7ssw.cn/
address=/7x-star.info/
address=/8020home.com.cn/
address=/805.red/
address=/815ss.cn/
address=/8pay.wang/
address=/964ka.cn/
address=/98hx.cn/
address=/aaayc.cn/
address=/abtao.wang/
address=/ahxhny.cn/
address=/aibantian.cn/
address=/aiia.xin/
address=/aiks.wang/
address=/aipu.mobi/
address=/aivento.cn/
address=/aiwenyisheng.mobi/
address=/aixintou.com.cn/
address=/amao.mobi/
address=/aup.mobi/
address=/baichuanbi.wang/
address=/barrister.org.cn/
address=/baseniao.com.cn/
address=/baxt.mobi/
address=/beiyu.xin/
address=/benniuluntai.cn/
address=/bjhjw.com.cn/
address=/blood23.cn/
address=/bzcjy.cn/
address=/calarm.info/
address=/callmewx.cn/
address=/cangshu.info/
address=/canwi.mobi/
address=/cdshusen.cn/
address=/cdxjt.mobi/
address=/chaoxianleather.ltd/
address=/chenyayun.com.cn/
address=/china-oxygen.cn/
address=/china99315.cn/
address=/chinae.mobi/
address=/chinapsj.com.cn/
address=/chinapulverizer.com.cn/
address=/chinaso.red/
address=/chinaxiedu.cn/
address=/codetips.wang/
address=/cqmjjx.cn/
address=/crystalmart.cn/
address=/cs-bailing.com.cn/
address=/cstmedia.com.cn/
address=/curtainsky.wang/
address=/cxlm.net.cn/
address=/cyp889.cn/
address=/cz4444.cn/
address=/dashantechan.cn/
address=/dat.red/
address=/dhouse.mobi/
address=/diaoguoshi.mobi/
address=/dinuojixie.com.cn/
address=/dnjj.mobi/
address=/dspack.com.cn/
address=/dzhss.cn/
address=/edmontonlife.info/
address=/eduace.com.cn/
address=/eyewand.cn/
address=/fadian.xin/
address=/fanjis.cn/
address=/fashion-hat.cn/
address=/fdkjt.cn/
address=/feiyun.info/
address=/fhfg.net.cn/
address=/fjs043.cn/
address=/fjs056.cn/
address=/forgot.mobi/
address=/freestudio.info/
address=/fy6x8o.cn/
address=/fzojq.info/
address=/getmos.cn/
address=/gjh111.cn/
address=/glnvdc.cn/
address=/gpscard.cn/
address=/greenprints.org.cn/
address=/gsgqwl.wang/
address=/gtlp.net.cn/
address=/gzjtfzs.cn/
address=/gzmcjt.cn/
address=/gzqczl.cn/
address=/gzsadlmy.cn/
address=/hanhooo.cn/
address=/haoduoyi1688.cn/
address=/haoeat.info/
address=/haoyangmao.ltd/
address=/haoyoushuo.cn/
address=/hbyinzhibao.cn/
address=/hccwwz.cn/
address=/heimi.red/
address=/helove.xyz/
address=/hktedu.site/
address=/hnwlyy.com.cn/
address=/hongze.info/
address=/hot-stories.cn/
address=/hskj88.cn/
address=/htnote.info/
address=/huanbao110.com.cn/
address=/huanbaoxiangmu.xyz/
address=/huangdao.info/
address=/huaqikonggu.com.cn/
address=/huaqiss.cn/
address=/huayiav.cn/
address=/huha.ink/
address=/huilian.info/
address=/hundun.mobi/
address=/hupuzhibo.cn/
address=/hygqtz.cn/
address=/hzdhr.cn/
address=/ib00.cn/
address=/imzhide.net.cn/
address=/iqyewu.cn/
address=/iyumiao.com.cn/
address=/japheth.com.cn/
address=/jbcbio.cn/
address=/jhbsq.cn/
address=/jingyixueyuan.cn/
address=/jinlanqiangyi.cn/
address=/jiuaixianzhi.mobi/
address=/jmait.cn/
address=/jmogo.cn/
address=/jnykjgs.cn/
address=/jpuv.cn/
address=/jqki.cn/
address=/jsjs.pro/
address=/judantech.site/
address=/jxqfu.cn/
address=/jxss88.mobi/
address=/kcvc.com.cn/
address=/kedeng.xin/
address=/king-oak.cn/
address=/kocom.mobi/
address=/kuaica.info/
address=/kuaidifeng.cn/
address=/l520.ltd/
address=/lafontainedessenterue.cn/
address=/lcr.kim/
address=/lcyt.info/
address=/ledian.pro/
address=/lightblue.red/
address=/lilangdianqi.cn/
address=/limkokwing-edu.cn/
address=/lindawei.cn/
address=/littlebee.site/
address=/liuguoyu.wang/
address=/lixincxy.cn/
address=/llanotextiles.cn/
address=/lningcity.com.cn/
address=/lnjseq.info/
address=/lnsbhzy.cn/
address=/lovecar.net.cn/
address=/lulumao.com.cn/
address=/lumeo.cn/
address=/luomanzhubao.cn/
address=/lvxingxian.cn/
address=/lwfw88.cn/
address=/lygnasa.cn/
address=/lytrjx.cn/
address=/lyzon.com.cn/
address=/meigeer.com.cn/
address=/menghuanzhilv.cn/
address=/mifun.mobi/
address=/murroliving.com.cn/
address=/myzhuanghe.cn/
address=/nankuan.xin/
address=/newweb.top/
address=/newwiesdom.com.cn/
address=/newzheng.cn/
address=/ngtraveler.com.cn/
address=/opai.red/
address=/opto-22.com.cn/
address=/oxi23.cn/
address=/pdiinfo.com.cn/
address=/pdsxp.cn/
address=/penglei.info/
address=/phjml.cn/
address=/pilipala.info/
address=/pszs388.cn/
address=/qhsyg.top/
address=/qianwei.wang/
address=/qichacha.ink/
address=/qincai.info/
address=/qishituan.top/
address=/qiyeit.com.cn/
address=/qkxlyg.cn/
address=/qmin.xin/
address=/qnvljz.info/
address=/qqwlfm.cn/
address=/qrtjwa.cn/
address=/samevay.com.cn/
address=/sapwells.info/
address=/scdcd333.cn/
address=/scfans.cn/
address=/scfw.wang/
address=/scifc.mobi/
address=/sdlzmm.cn/
address=/sdyongyan.com.cn/
address=/shanyi.info/
address=/shinedaily.cn/
address=/shkunjia.com.cn/
address=/shoujiawang.cn/
address=/shouyili.mobi/
address=/sjdjcn.cn/
address=/smart-way2.com.cn/
address=/smmx3.cn/
address=/sobin.wang/
address=/spreadable.com.cn/
address=/sscjchina.com.cn/
address=/steeltrader.com.cn/
address=/sunderport.com.cn/
address=/suntechauto.com.cn/
address=/sxjcjdc.cn/
address=/sxltfj.cn/
address=/szmpc.cn/
address=/taihe2002.cn/
address=/tanzhen.info/
address=/taogou.site/
address=/tastevision.cn/
address=/techkey.com.cn/
address=/tianhuicun.com.cn/
address=/titan-solar.cn/
address=/tkmly.cn/
address=/todayjiaxiang.cn/
address=/tradesoul.cn/
address=/tuoens.cn/
address=/txtxz.org.cn/
address=/udr26c.cn/
address=/uk8866.cn/
address=/vaniok.cn/
address=/vpkq.cn/
address=/wangjinhu.wang/
address=/wangzhichao.info/
address=/weilang.site/
address=/wimaxnetworks.cn/
address=/wojiacanting.info/
address=/wpe.red/
address=/wtorain.red/
address=/wucheng.info/
address=/wzfjsh.cn/
address=/wzhagc.cn/
address=/x1ka.cn/
address=/xcvf.info/
address=/xiaocai-rookie.info/
address=/xiaomeihq.info/
address=/xinshengchuanmei.cn/
address=/xinyikeji.red/
address=/xmglass.cn/
address=/xuexingkeji.cn/
address=/xxdlg.cn/
address=/xz518.cn/
address=/xztyzs.cn/
address=/yanhao.red/
address=/yaxujiancai.cn/
address=/ych168.cn/
address=/ycreateam.cn/
address=/ycx.kim/
address=/yglhcn.cn/
address=/ygpd.wang/
address=/yimingxiang.com.cn/
address=/yiqifaxian.wang/
address=/yiqiu.mobi/
address=/yisheng120.info/
address=/yizhongyi.info/
address=/ynyfcz.cn/
address=/yppw666.cn/
address=/yucefa.cn/
address=/yw78.cn/
address=/yybeast.mobi/
address=/zghs.net.cn/
address=/zgjckgys.com.cn/
address=/zhaowaibao.mobi/
address=/zhougong.info/
address=/zhuren.site/
address=/zioe.com.cn/
address=/ziyouxiaoyuan.cn/
address=/zq-hk.cn/
address=/zsdzcpw.mobi/
address=/zswhcsfww.mobi/
address=/zszgjiejuw.mobi/
address=/zzasj.cn/
address=/zzhssy.cn/

34360
dist/anti-ad-full.conf vendored

File diff suppressed because it is too large Load Diff

@ -10,30 +10,7 @@
*/
class addressMaker{
/**
* http get 方法,一般用于下载文件
*
* @param $url
* @return bool|string
*/
public static function http_get($url){
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HTTPGET, 1);
curl_setopt($ch, CURLOPT_TIMEOUT, 30);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT, '^_^ angent 2.2.5/' . phpversion());
$result = curl_exec($ch);
$errno = curl_errno($ch);
curl_close($ch);
return $result;
}
class addressMaker {
/**
* 分离域名
@ -60,7 +37,7 @@ class addressMaker{
$str_reg .= ')';
$str_reg .= '(\.cn|\.hk|\.tw|\.uk|\.jp|\.kr|\.th|\.au|\.ua|\.so|\.br|\.sg|\.pt|\.ec|\.ar|\.my|\.tr|\.bd|\.mk|\.za|\.mt)?)$/';
if(preg_match($str_reg, $str_domain,$matchs)){
if(preg_match($str_reg, $str_domain, $matchs)){
return strval($matchs[1]);
}
@ -82,12 +59,12 @@ class addressMaker{
$str_easylist = $str_easylist . "\n"; //防止最后一行没有换行符
$i=0;
$i = 0;
$arr_domains = array();
while($i < $strlen){
$end_pos = strpos($str_easylist, "\n", $i);
$line = trim(substr($str_easylist, $i, $end_pos - $i));
$i = $end_pos+1;
$i = $end_pos + 1;
if(empty($line) || strlen($line) < 3){
continue;
}
@ -125,12 +102,12 @@ class addressMaker{
$str_hosts = $str_hosts . "\n"; //防止最后一行没有换行符
$i=0;
$i = 0;
$arr_domains = array();
while($i < $strlen){
$end_pos = strpos($str_hosts, "\n", $i);
$line = trim(substr($str_hosts, $i, $end_pos - $i));
$i = $end_pos+1;
$i = $end_pos + 1;
if(empty($line) || ($line{0} == '#')){//注释行忽略
continue;
}
@ -153,16 +130,11 @@ class addressMaker{
return $arr_domains;
}
public static function write_to_conf($arr_result, $str_file, $q_file){
public static function write_to_conf($arr_result, $str_file){
$fp = fopen($str_file, 'w');
$fp2 = fopen($q_file, 'w');
$write_len = fwrite($fp, '#TIME=' . date('YmdHis'). "\n");
$write_len = fwrite($fp, '#TIME=' . date('YmdHis') . "\n");
$write_len += fwrite($fp, '#URL=https://github.com/gentlyxu/anti-AD' . "\n");
fwrite($fp2, '[TCP]' . "\n");
fwrite($fp2, 'USER-AGENT,com.apple.*,DIRECT' . "\n");
fwrite($fp2, 'USER-AGENT,FindMy*,DIRECT' . "\n");
fwrite($fp2, 'USER-AGENT,Maps*,DIRECT' . "\n");
foreach($arr_result as $rk => $rv){
@ -180,7 +152,6 @@ class addressMaker{
continue;
}
$write_len += fwrite($fp, 'address=/' . $rv . '/' . "\n");
fwrite($fp2, 'HOST-SUFFIX,' . $rv . ',REJECT' . "\n");
continue;
}
@ -188,7 +159,6 @@ class addressMaker{
if(in_array('.' . $rk, $rv) || in_array('www.' . $rk, $rv) || in_array($rk, $rv)){
$write_len += fwrite($fp, 'address=/' . $rk . '/' . "\n");
fwrite($fp2, 'HOST-SUFFIX,' . $rk . ',REJECT' . "\n");
continue;
}
@ -212,7 +182,6 @@ class addressMaker{
continue;
}
$write_len += fwrite($fp, 'address=/' . implode('.', $tmp_arr2) . '/' . "\n");
fwrite($fp2, 'HOST-SUFFIX,' . implode('.', $tmp_arr2) . ',REJECT' . "\n");
}
$written_flag = true;
break;
@ -226,12 +195,10 @@ class addressMaker{
$arr_written[] = $rvv;
$write_len += fwrite($fp, 'address=/' . $rvv . '/' . "\n");
fwrite($fp2, 'HOST-SUFFIX,' . $rvv . ',REJECT' . "\n");
}
}
fclose($fp);
fclose($fp2);
return $write_len;
}

@ -0,0 +1,66 @@
<?php
/**
* 工具类
*
* @date 2019.12.14
* @author gently
*/
class utils {
/**
* http get 方法,一般用于下载文件
*
* @param $url
* @return bool|string
*/
public static function http_get($url){
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HTTPGET, 1);
curl_setopt($ch, CURLOPT_TIMEOUT, 30);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT, '^_^ angent 2.2.5/' . phpversion());
$result = curl_exec($ch);
$errno = curl_errno($ch);
curl_close($ch);
return $result;
}
/**
* 数组合并将相同key的值相加而不是生成数组或合并不同key将全部保留
*
* @param $arr1
* @param $arr2
* @return array
*/
public static function array_merge_plus($arr1, $arr2){
if(!is_array($arr1)){
$arr1 = array();
}
if(!is_array($arr2)){
$arr2 = array();
}
$arr1 = array_merge_recursive($arr1, $arr2);
$arr_result = array();
foreach($arr1 as $key => $val){
$arr_result[$key] = array();
if(!is_array($val)){
continue;
}
foreach($val as $k => $v){
$arr_result[$key][$k] = is_array($v) ? array_sum($v) : $v;
}
}
return $arr_result;
}
}

@ -17,8 +17,8 @@ if(PHP_SAPI != 'cli'){
die('nothing.');
}
require('./lib/addressMaker.class.php');
$arr_blacklist = require('./black_domain_list.php');
$arr_whitelist = require('./white_domain_list.php');
$arr_blacklist = require('./lib/black_domain_list.php');
$arr_whitelist = require('./lib/white_domain_list.php');
$arr_result = array();
@ -37,7 +37,6 @@ $arr_result = array_merge_recursive($arr_result, addressMaker::get_domain_from_e
$host1 = file_get_contents('./origin-files/hosts1');
$arr_result = array_merge_recursive($arr_result, addressMaker::get_domain_list($host1));
// $host2 = makeAddr::http_get('http://www.malwaredomainlist.com/hostslist/hosts.txt');
$host2 = file_get_contents('./origin-files/hosts2');
$arr_result = array_merge_recursive($arr_result, addressMaker::get_domain_list($host2));

File diff suppressed because it is too large Load Diff

@ -0,0 +1,83 @@
<?php
/**
* 重新整理列表分离出basicfull剔除失效域名
*
* @file research-addr.php
* @author gently
* @date 2019.12.14
*
*/
set_time_limit(600);
error_reporting(0);
if(PHP_SAPI != 'cli'){
die('nothing.');
}
require dirname(__DIR__) . '/lib/utils.class.php';
define('ORIGIN_DIR', dirname(__DIR__) . '/origin-files/');
define('DIST_DIR', dirname(__DIR__) . '/dist/');
define('LIB_DIR', dirname(__DIR__) . '/lib/');
define('SRC_FILE', dirname(__DIR__) . '/adblock-for-dnsmasq.conf');
$arr_dead_horse = array();
$arr_china_list = array();
$dead_horse_files = glob(ORIGIN_DIR . 'dead-horse_*');
foreach($dead_horse_files as $f){
$arr_content = require_once $f;
$arr_dead_horse = utils::array_merge_plus($arr_dead_horse, $arr_content);
}
$china_list_files = glob(ORIGIN_DIR . 'china-list_*');
foreach($china_list_files as $f){
$arr_content = require_once $f;
$arr_china_list = array_merge($arr_china_list, $arr_content);
}
if(count($arr_china_list) <= 0 && count($arr_dead_horse) <= 0){
die('empty filter list');
}
//黑名单在basic中也是必须的
$black_list = require LIB_DIR . '/black_domain_list.php';
$src_fp = fopen(SRC_FILE, 'r');
$basic_fp = fopen(DIST_DIR . '/anti-ad-basic.conf', 'w');
$full_fp = fopen(DIST_DIR . '/anti-ad-full.conf', 'w');
$write_len = fwrite($basic_fp, '#TIME=' . date('YmdHis') . "\n");
$write_len += fwrite($basic_fp, '#URL=https://github.com/gentlyxu/anti-AD' . "\n");
$write_len = fwrite($full_fp, '#TIME=' . date('YmdHis') . "\n");
$write_len += fwrite($full_fp, '#URL=https://github.com/gentlyxu/anti-AD' . "\n");
while(!feof($src_fp)){
$row = fgets($src_fp, 512);
if(empty($row)){
continue;
}
if(preg_match('/^address=\/(.+)?\/$/', $row, $matches)){
if(array_key_exists($matches[1], $arr_dead_horse)){
if(isset($arr_dead_horse[$matches[1]]['dead']) && $arr_dead_horse[$matches[1]]['dead'] > 2){
continue;
}elseif(isset($arr_dead_horse[$matches[1]]['empty']) && $arr_dead_horse[$matches[1]]['empty'] > 3){
continue;
}elseif(isset($arr_dead_horse[$matches[1]]['problem']) && $arr_dead_horse[$matches[1]]['problem'] > 3){
continue;
}
}
fwrite($full_fp, "address=/{$matches[1]}/\n");
if(array_key_exists($matches[1], $arr_china_list) || array_key_exists($matches[1], $black_list)){
fwrite($basic_fp, "address=/{$matches[1]}/\n");
}
}
}
//删除之前验证工具生成的中间结果
foreach(array_merge($china_list_files, $dead_horse_files) as $f){
// unlink($f);
}

@ -88,9 +88,12 @@ while (!feof($src_fp)) {
$dead_horse[$matches[1]]['empty']++;
}
} catch (Net_DNS2_Exception $e) {
//3=dns记录不存在
if ($e->getCode() == 3) {
$dead_horse[$matches[1]]['dead']++;
$dead_horse[$matches[1]]['dead']++;//3=dns记录不存在
}elseif($e->getCode() == 2) {
$dead_horse[$matches[1]]['problem']++;//2=查询失败dns服务器没有返回正确记录
}elseif($e->getCode() == 203){
$dead_horse[$matches[1]]['timeout']++;//203=查询超时
} else {
echo date('m-d H:i:s'), "[", $matches[1], "]", $e->getMessage(), ",code:", $e->getCode(), "\n";
}
@ -107,4 +110,4 @@ try {
echo date('m-d H:i:s'), "write file failed:", $e->getMessage(), "\t", $e->getCode(), "\n";
}
echo 'Time cost:', microtime(true) - START_TIME, "s\n";
echo 'Time cost:', microtime(true) - START_TIME, "s, at", date('m-d H:i:s'), "\n";

Loading…
Cancel
Save