Autor Tópico: Googlebots & Spiders  (Lida 10233 vezes)

0 Membros e 2 Visitantes estão a ver este tópico.

Offline Darkness

  • VectorSeries.Net
  • Administrador
  • *
  • Mensagens: 337
  • Rank: 3
  • Sexo: Masculino
  • Vector Series
    • Vector Series
Googlebots & Spiders
« em: 03 de Agosto de 2009, 05:31 »
./index.php

Encontre:
Código: [Seleccione]
// Load the settings from the settings table, and perform operations like optimizing.
reloadSettings();

Adcione Após:
Código: [Seleccione]
//Get rid of ?PHPSESSID in the case is a Googlebot any other Spider. Even if is a user (maybe User-Agent extension), will be redirected. Easier this way.
if ($modSettings['ob_googlebot_redirect_phpsessid'] && ob_googlebot_getAgent($_SERVER['HTTP_USER_AGENT'], $spider_name, $agent))
{
$actualurl = 'http://'.$_SERVER['HTTP_HOST'].$_SERVER['REQUEST_URI'];
$correcturl = preg_replace('/([?&]PHPSESSID=[^&]*)/', '', $actualurl);
$correcturl = str_replace('index.php&', 'index.php?', $correcturl);

if ($correcturl != $actualurl) {
header("HTTP/1.1 301 Moved Permanently");
header("Location: " . $correcturl);
exit();
}
}

./SSI.php

Encontre:
Código: [Seleccione]
// Load the users online right now.
$result = db_query("

Adcione Antes:
Código: [Seleccione]
global $modSettings;
Encontre:
Código: [Seleccione]
lo.ID_MEMBER, lo.logTime,
Adcione Após:
Código: [Seleccione]
lo.url,
Encontre:
Código: [Seleccione]
while ($row = mysql_fetch_assoc($result))
{
if (!isset($row['realName']))

Adcione Antes:
Código: [Seleccione]
$return['spiders'] = array();
$return['num_spiders'] = 0;

Encontre:
Código: [Seleccione]
$return['guests']++;
Substitua Por:
Código: [Seleccione]
{
// Get the request parameters..
$actions = @unserialize($row['url']);

// Is a spider?
$is_spider = ob_googlebot_getAgent($actions['USER_AGENT'], $spider_name, $agent, $row['ID_MEMBER'] == 0);

if (!$is_spider)
$return['guests']++;
else
{
$return['num_spiders']++;

if ($modSettings['ob_googlebot_display_agent'])
$spider_name = $agent;

if ($modSettings['ob_googlebot_count_all_instances'] && $modSettings['ob_googlebot_display_all_instances'])
$return['spiders'][] = $spider_name;
else
$return['spiders'][$agent] = $spider_name;
}
}

Encontre:
Código: [Seleccione]
if (!empty($return['users']))
{
krsort($return['users']);
$userlist = array_keys($return['users']);
$return['users'][$userlist[count($userlist) - 1]]['is_last'] = true;
}

Adcione Após:
Código: [Seleccione]
// Sort spiders list
ksort($return['spiders']);

// Allowed user to see spiders online?
if (!allowedTo('googlebot_view'))
{
$return['spiders'] = array();
$return['guests'] += $return['num_spiders'];
}

// Don't count all instances of a spider, only 1 for each different spider
if (!empty($return['spiders']) && !$modSettings['ob_googlebot_count_all_instances'])
$return['num_spiders'] = count($return['spiders']);

// Diplay how many instances of each spider
if (!empty($return['spiders']) && $modSettings['ob_googlebot_count_all_instances'] && $modSettings['ob_googlebot_display_all_instances'])
{
$spider_grouped = array_count_values($return['spiders']);

$return['spiders'] = array();
foreach ($spider_grouped as $k => $v)
$return['spiders'][$k] = $k . ($v == 1 ? '' : ' (' . $v . ')');
}

Encontre:
Código: [Seleccione]
$return['total_users'] = $return['num_users'] + $return['guests']
Adcione Após:
Código: [Seleccione]
+ ($modSettings['ob_googlebot_count_most_online'] ? $return['num_spiders'] : 0)
Encontre:
Código: [Seleccione]
$return['guests'], ' ', $return['guests'] == 1
Adcione Antes:
Código: [Seleccione]
(empty($return['spiders']) ? '' : $return['num_spiders'] . ' ' . ($return['num_spiders'] == 1 ? $txt['ob_googlebot_spider'] : $txt['ob_googlebot_spiders']) . ', '),
Encontre:
Código: [Seleccione]
foreach ($return['users'] as $user)
echo $user['hidden'] ? '<i>' . $user['link'] . '</i>' : $user['link'], $user['is_last'] ? '' : ', ';

Adcione Após:
Código: [Seleccione]
if (!empty($return['spiders']))
{
if ($modSettings['ob_googlebot_display_own_list'])
echo '
<br />';
else
{
if (!empty($return['users']))
echo ', ';
}

echo implode(', ', $return['spiders']);
}

./Sources/BoardIndex.php

Encontre:
Código: [Seleccione]
// Load the users online right now.
$result = db_query("
SELECT
lo.ID_MEMBER, lo.logTime, mem.realName, mem.memberName, mem.showOnline,

Substitua Por:
Código: [Seleccione]
// Load the users online right now.
$result = db_query("
SELECT
lo.ID_MEMBER, lo.logTime, lo.url, mem.realName, mem.memberName, mem.showOnline,

Encontre:
Código: [Seleccione]
$context['num_users_hidden'] = 0;
Adcione Após:
Código: [Seleccione]
$context['spiders'] = array();
$context['num_spiders'] = 0;

Encontre:
Código: [Seleccione]
$context['num_guests']++;
continue;

Substitua Por:
Código: [Seleccione]
// Get the request parameters..
$actions = @unserialize($row['url']);

// Is a spider?
$is_spider = ob_googlebot_getAgent($actions['USER_AGENT'], $spider_name, $agent);

if (!$is_spider)
$context['num_guests']++;
else
{
$context['num_spiders']++;

if ($modSettings['ob_googlebot_display_agent'])
$spider_name = $agent;

if ($modSettings['ob_googlebot_count_all_instances'] && $modSettings['ob_googlebot_display_all_instances'])
$context['spiders'][] = $spider_name;
else
$context['spiders'][$agent] = $spider_name;
}

continue;

Encontre:
Código: [Seleccione]
$context['num_users_online'] = count($context['users_online']) + $context['num_users_hidden'];
Substitua Por:
Código: [Seleccione]
ksort($context['spiders']);

$context['num_users_online'] = count($context['users_online']) + $context['num_users_hidden'];

// Allowed user to see spiders online?
if (!allowedTo('googlebot_view'))
$context['spiders'] = array();

// Don't count all instances of a spider, only 1 for each different spider
if (!empty($context['spiders']) && !$modSettings['ob_googlebot_count_all_instances'])
$context['num_spiders'] = count($context['spiders']);

// Diplay how many instances of each spider
if (!empty($context['spiders']) && $modSettings['ob_googlebot_count_all_instances'] && $modSettings['ob_googlebot_display_all_instances'])
{
$spider_grouped = array_count_values($context['spiders']);

$context['spiders'] = array();
foreach ($spider_grouped as $k => $v)
$context['spiders'][$k] = $k . ($v == 1 ? '' : ' (' . $v . ')');
}

Encontre:
Código: [Seleccione]
$total_users = $context['num_guests'] +
Adcione Após:
Código: [Seleccione]
($modSettings['ob_googlebot_count_most_online'] ? $context['num_spiders'] : 0) +
Encontre:
Código: [Seleccione]
// Set the latest member.
Adcione Antes:
Código: [Seleccione]
// Allowed user to see spiders online (We change stuff here AFTER stats update). Spiders are displayed as guests again.
if (!allowedTo('googlebot_view'))
$context['num_guests'] += $context['num_spiders'];

./Sources/ManagePermissions.php

Encontre:
Código: [Seleccione]
'profile_remote_avatar' => false,
)

Adcione Após:
Código: [Seleccione]
,
'googlebot' => array(
'googlebot_view' => false,
)

./Sources/ModSettings.php

Encontre:
Código: [Seleccione]
$context['sub_template'] = 'show_settings';

$subActions = array(

Adcione Após:
Código: [Seleccione]
'googlebot' => 'ModifyGooglebotSettings',

Encontre:

Código: [Seleccione]
require_once($sourcedir . '/ManageServer.php');

$subActions = array(

Adcione Após:
Código: [Seleccione]
'googlebot' => 'ModifyGooglebotSettings',
Encontre:
Código: [Seleccione]
'karma' => array(
'title' => $txt['smf293'],
'href' => $scripturl . '?action=featuresettings;sa=karma;sesc=' . $context['session_id'],

Adcione Após:
Código: [Seleccione]
),
'googlebot' => array(
'title' => $txt['ob_googlebot_modname'],
'href' => $scripturl . '?action=featuresettings;sa=googlebot;sesc=' . $context['session_id'],

Encontre:
Código: [Seleccione]
$context['post_url'] = $scripturl . '?action=featuresettings2;save;sa=karma';
$context['settings_title'] = $txt['smf293'];

prepareDBSettingContext($config_vars);
}

Adcione Após:
Código: [Seleccione]
function ModifyGooglebotSettings()
{
global $txt, $scripturl, $context, $settings, $sc;

$config_vars = array(
// Count all instances of spiders?
array('check', 'ob_googlebot_count_all_instances'),
array('check', 'ob_googlebot_display_all_instances'),
array('check', 'ob_googlebot_display_agent'),
array('check', 'ob_googlebot_display_own_list'),
'',
// Count spiders on most online?
array('check', 'ob_googlebot_count_most_online'),
'',
// Redirect PHPSESSID URLs?
array('check', 'ob_googlebot_redirect_phpsessid'),
);

// Saving?
if (isset($_GET['save']))
{
saveDBSettings($config_vars);
redirectexit('action=featuresettings;sa=googlebot');
}

$context['post_url'] = $scripturl . '?action=featuresettings2;save;sa=googlebot';
$context['settings_title'] = $txt['ob_googlebot_modname'];

prepareDBSettingContext($config_vars);
}

./Sources/Subs.php

Encontre:
Código: [Seleccione]
function template_footer()
{
global $context, $settings, $modSettings, $time_start, $db_count;

Adcione Após:
Código: [Seleccione]
global $user_info, $db_prefix;

//Googlebot stats - Get info for this page
if ($modSettings['ob_googlebot_stats'])
{
$ob_googlebot_start = isset($context['start'])?$context['start']:'0';
if (isset($context['current_topic'])) // We are on a topic
{
$key = 'topic';
$value = "$context[current_topic].$ob_googlebot_start";
}
elseif (isset($context['current_board'])) // We are on a board
{
$key = 'board';
$value = "$context[current_board].$ob_googlebot_start";
}
else // We are somewhere else
{
$key = 'url';

// Remove PHPSESSID, just in case that the user has it in his URL, to avoid duplicates
$value = "$_SERVER[REQUEST_URI]";
$value = preg_replace('/([?&]PHPSESSID=[^&]*)/', '', $value);
$value = str_replace('index.php&', 'index.php?', $value);
}

$result = db_query("
SELECT lastvisit, frequency, visits
FROM {$db_prefix}ob_googlebot_stats
WHERE $key = '$value'
AND agent = 'Googlebot'
LIMIT 1", __FILE__, __LINE__);
if (mysql_num_rows($result) == 0) //No stats yet.
$lastvisit = $frequency = $visits = 0;
else
list ($lastvisit, $frequency, $visits) = mysql_fetch_row($result);
mysql_free_result($result);

// Is Googlebot? if so, we will update the stats BEFORE displaying it.
if (ob_googlebot_getAgent($_SERVER['HTTP_USER_AGENT'], $spider_name, $agent))
if ($agent == 'Googlebot')
{
$now = time();
$visits++;

if ($visits == 1) // First time, insert new record
db_query("
INSERT INTO {$db_prefix}ob_googlebot_stats
($key, visits, lastvisit, agent)
VALUES
('$value', $visits, $now, '$agent')
", __FILE__, __LINE__);
else
{
if ($visits == 2)
$frequency = $now - $lastvisit; /* This is the SECOND time Google visits this page */
else
$frequency = (($frequency * ($visits - 1)) + ($now - $lastvisit)) / $visits;

db_query("
UPDATE {$db_prefix}ob_googlebot_stats
SET frequency = $frequency, visits = $visits, lastvisit = $now
WHERE $key = '$value'
AND agent = 'Googlebot'
LIMIT 1", __FILE__, __LINE__);
}

$lastvisit = $now;
}

if ($visits > 0)
$context['ob_googlebot_stats'] = array(
'Googlebot' => array('frequency' => $frequency, 'visits' => $visits, 'lastvisit' => $lastvisit)
);
}

Encontre (No fim do Arquivo):
Código: [Seleccione]
?>
Adcione Antes:
Código: [Seleccione]
//Function to check if the user-agent provided belongs to a spider. Based on getAgent function made by Owdy.
function ob_googlebot_getAgent(&$user_agent, &$spider_name, &$result)
{
$known_spiders = array (
//Search Spiders
array (
'agent' => 'WISENutbot',
'spidername' => 'Looksmart spider',
),
array (
'agent' => 'MSNBot',
'spidername' => 'MSN spider',
),
array (
'agent' => 'W3C_Validator',
'spidername' => 'W3C Validator',
),
array (
'agent' => 'Googlebot-Image',
'spidername' => 'Google-Image Spider',
),
array (
'agent' => 'Googlebot',
'spidername' => 'Google spider',
),
array (
'agent' => 'Mediapartners-Google',
'spidername' => 'Google AdSense spider',
),
array (
'agent' => 'Openbot',
'spidername' => 'Openfind spider',
),

array (
'agent' => 'Yahoo! Slurp',
'spidername' => 'Yahoo spider',
),
array (
'agent' => 'FAST-WebCrawler',
),
array (
'agent' => 'Wget',
),
array (
'agent' => 'Ask Jeeves',
),
array (
'agent' => 'Speedy Spider',
),
array (
'agent' => 'SurveyBot',
),
array (
'agent' => 'IBM_Planetwide',
),
array (
'agent' => 'GigaBot',
),
array (
'agent' => 'ia_archiver',
),
array (
'agent' => 'FAST-WebCrawler',
),
array (
'agent' => 'Inktomi Slurp',
),
array (
'agent' => 'appie',
'spidername' => 'Walhello spider',
),
array (
            'agent' => 'FeedBurner/1.0',
            'spidername' => 'Feedburner',
        ),
        array (
            'agent' => 'Feedfetcher-Google',
        ),
        array (
            'agent' => 'OmniExplorer_Bot/6.68',
            'spidername' => 'OmniExplorer Bot',
        ),
        array (
            'agent' => 'http://www.relevantnoise.com',
            'spidername' => 'relevantNOISE',
        ),
        array (
            'agent' => 'NewsGatorOnline/2.0',
            'spidername' => 'NewsGatorOnline',
        ),
        array (
            'agent' => 'ping.blo.gs/2.0',
        ),
        array (
            'agent' => 'Jakarta Commons-HttpClient/3.0.1',
            'spidername' => 'Amazon',
        ),
          array (
            'agent' => 'Jakarta Commons-HttpClient/3.0-rc2',
            'spidername' => 'Amazon',
        ),
);

foreach($known_spiders AS $poss)
if (strpos(strtolower($user_agent), strtolower($poss['agent'])) !== false)
{
$spider_name = isset($poss['spidername']) ? $poss['spidername'] : $poss['agent'];
$result = $poss['agent'];
return true;
}

return false;
}

./Themes/default/languages/Modifications.english.php

Encontre: (no fim do arquivo):
Código: [Seleccione]
?>
Adcione Antes:
Código: [Seleccione]
// OB - Googlebot - Begin

// Boardindex Strings
$txt['ob_googlebot_modname'] = 'Googlebot & Spiders';
$txt['ob_googlebot_spider'] = 'Spider';
$txt['ob_googlebot_spiders'] = 'Spiders';
$txt['ob_googlebot_spiders_last_active'] = 'Spiders active in past ' . $modSettings['lastActive'] . ' minutes';

// ModSettings
$txt['ob_googlebot_count_all_instances'] = 'Count all instances of same spider';
$txt['ob_googlebot_display_all_instances'] = 'Display all instances of same spider <div class="smalltext">("' . $txt['ob_googlebot_count_all_instances'] . '" must be selected)</div>';
$txt['ob_googlebot_display_agent'] = 'Display agent instead of name';
$txt['ob_googlebot_display_own_list'] = 'Display spiders in its own list';
$txt['ob_googlebot_count_most_online'] = 'Count in "Most Online"';
$txt['ob_googlebot_redirect_phpsessid'] = 'Redirect PHPSESSID URLs';

// Stats
$txt['ob_googlebot_stats_lastvisit'] = 'Google visited last this page ';

// Permissions
$txt['permissiongroup_googlebot'] = $txt['ob_googlebot_modname'];
$txt['permissionname_googlebot_view'] = 'View Googlebot & Spiders';

// OB - Googlebot - End

./Themes/default/languages/Modifications.portuguese_pt.php (Ou a versão que usa)

Encontre (no fim do arquivo):
Código: [Seleccione]
?>
Adcione Antes:
Código: [Seleccione]
// OB - Googlebot - Begin translated By Darkness

// Boardindex Strings
$txt['ob_googlebot_modname'] = 'Googlebot & Spiders';
$txt['ob_googlebot_spider'] = 'Spider';
$txt['ob_googlebot_spiders'] = 'Spiders';
$txt['ob_googlebot_spiders_last_active'] = 'Spiders ativos nos últimos ' . $modSettings['lastActive'] . ' minutos';

// ModSettings
$txt['ob_googlebot_count_all_instances'] = 'Contar todas as instâncias do mesmo Spider';
$txt['ob_googlebot_display_all_instances'] = 'Exibir todas as instâncias do mesmo Spider <div class="smalltext">("' . $txt['ob_googlebot_count_all_instances'] . '" deve ser selecionado)</div>';
$txt['ob_googlebot_display_agent'] = 'Nome de exibição e o uso do agente';
$txt['ob_googlebot_display_own_list'] = 'Exibir os Spiders na sua própria lista';
$txt['ob_googlebot_count_most_online'] = 'Contagem no "Mais onlines"';
$txt['ob_googlebot_redirect_phpsessid'] = 'Redirecionar PHPSESSID URLs';

// Stats
$txt['ob_googlebot_stats_lastvisit'] = 'Google finalmente visitou esta página ';

// Permissions
$txt['permissiongroup_googlebot'] = $txt['ob_googlebot_modname'];
$txt['permissionname_googlebot_view'] = 'Visualizar Googlebot & Spiders';

// OB - Googlebot - End Translated By Darkness

./Themes/default/index.template.php (Se necessário também no Thema que usas)

Encontre:
Código: [Seleccione]
if ($context['show_load_time'])
echo '
<span class="smalltext">', $txt['smf301'], $context['load_time'], $txt['smf302'], $context['load_queries'], $txt['smf302b'], '</span>';

Adcione Após:
Código: [Seleccione]
if (isset($context['ob_googlebot_stats']))
echo '
<br /><br /><span class="smalltext">', $txt['ob_googlebot_stats_lastvisit'], timeformat($context['ob_googlebot_stats']['Googlebot']['lastvisit']), '</span>';


./Themes/default/BoardIndex.template.php
(Se Necessário também no thema que usas)

Encontre:
Código: [Seleccione]
$context['num_guests'], ' ', $context['num_guests'] == 1


Adcione Antes:
Código: [Seleccione]
(empty($context['spiders']) ? '' : $context['num_spiders'] . ' ' . ($context['num_spiders'] == 1 ? $txt['ob_googlebot_spider'] : $txt['ob_googlebot_spiders']) . ', '),
Encontre:
Código: [Seleccione]
', $txt[140], ':<br />', implode(', ', $context['list_users_online']);
Adcione Após:
Código: [Seleccione]
if (!empty($context['spiders']))
{
if ($modSettings['ob_googlebot_display_own_list'])
echo '
<br />
', $txt['ob_googlebot_spiders_last_active'], ':<br />';
else
{
if (empty($context['users_online']))
echo '
', $txt[140], ':<br />';
else
echo ', ';
}

echo implode(', ', $context['spiders']);
}

Translator SMF - Brazilian
Não respondo MP de ajuda, Poste sua duvida no forum para ajuda

Offline Webmaster

  • Desenvolvimento
  • *
  • Mensagens: 2236
  • Rank: 113
  • Sexo: Masculino
  • JoomlaMZ !
    • Servidores
Re:Googlebots & Spiders
« Responder #1 em: 03 de Agosto de 2009, 09:14 »
Muito Bom
Bom trabalho
Apllic! WebSuporte

Offline M. Kazka

  • Suporte
  • *
  • Mensagens: 299
  • Rank: 1
  • Sexo: Masculino
Re:Googlebots & Spiders
« Responder #2 em: 03 de Agosto de 2009, 20:26 »
Muita coisa mesmo, mais eu testei e deu erro ainda ¬¬'

depois tento de novo.....
« Última modificação: 05 de Agosto de 2009, 00:24 por M. Kazka »