<?php
/**
* Detects the language of a given piece of text.
*
* Attempts to detect the language of a sample of text by correlating ranked
* 3-gram frequencies to a table of 3-gram frequencies of known languages.
*
* PHP versions 4 and 5
*
* @category Text
* @package LanguageDetect
* @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
* @copyright 2005 Nicholas Pisarro
* @license http://www.debian.org/misc/bsd.license BSD
* @link http://junker.org/~nickp/langdetect
* @version CVS: $Id: LanguageDetect.php,v 1.33 2005/12/29 18:36:15 nickp Exp $
*/
require_once '/home/nickp/share/pear/PEAR.php';
/**
* Language detection class
*
* Requires the langauge model database (lang.dat) that should have
* accompanied this class definition in order to be instantiated.
*
* @category Text
* @package LanguageDetect
* @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
* @copyright 2005 Nicholas Pisarro
* @license http://www.debian.org/misc/bsd.license BSD
* @link http://junker.org/~nickp/langdetect
* @version Release: @package_version@
* @todo allow users to generate their own language models
*/
class Text_LanguageDetect
{
/**
* The filename that stores the trigram data for the detector
*
* @var string
* @access private
*/
var $_db_filename = 'lang.dat';
/**
* The data directory
*
* @var string
* @access private
*/
var $_data_dir = '@data_dir@';
/**
* The size of the trigram data arrays
*
* @var int
* @access private
*/
var $_threshold = 300;
/**
* The trigram data for comparison
*
* Will be loaded on start from $this->_db_filename
*
* May be set to a PEAR_Error object if there is an error during its
* initialization
*
* @var array
* @access private
*/
var $_lang_db = array();
/**
* Whether or not to simulate perl's Language::Guess exactly
*
* @access private
* @var bool
* @see setPerlCompatible()
*/
var $_perl_compatible = false;
/**
* the maximum possible score.
*
* needed for score normalization. Different depending on the
* perl compatibility setting
*
* @access private
* @var int
* @see setPerlCompatible()
*/
var $_max_score = 0;
/**
* Constructor
*
* Will attempt to load the language database. If it fails, you will get
* a PEAR_Error object returned when you try to use detect()
*
*/
function Text_LanguageDetect()
{
$this->_lang_db = $this->_readdb($this->_get_db_loc());
}
/**
* Returns the path to the location of the database
*
* @access private
* @return string expected path to the language model database
*/
function _get_db_loc()
{
// checks if this has been installed properly
if ($this->_data_dir != '@' . 'data_dir' . '@') {
// if the data dir was set by the PEAR installer, use that
return $this->_data_dir . '/Text_LanguageDetect/' . $this->_db_filename;
} else {
// try the local working directory if otherwise
return '../data/' . $this->_db_filename;
}
}
/**
* Loads the language trigram database from filename
*
* Trigram datbase should be a serialize()'d array
*
* @access private
* @param string $fname the filename where the data is stored
* @return array the language model data
* @throws PEAR_Error
*/
function _readdb($fname)
{
if (!file_exists($fname) || !is_readable($fname)) {
return PEAR::raiseError("Language database does not exist or is not readable.");
}
if (function_exists('file_get_contents')) {
return unserialize(file_get_contents($fname));
} else {
// if you don't have file_get_contents(), then this is the next fastest way
ob_start();
readfile($fname);
$contents = ob_get_contents();
ob_end_clean();
return unserialize($contents);
}
}
/**
* Checks if this object is ready to detect languages
*
* @access private
* @param mixed &$err error object to be returned by reference, if any
* @return bool true if no errors
*/
function _setup_ok(&$err)
{
if (PEAR::isError($this->_lang_db)) {
// if there was an error from when the language database was loaded
// then return that error
$err = $this->_lang_db;
return false;
} elseif (!is_array($this->_lang_db)) {
$err = PEAR::raiseError('Language database is not an array.');
return false;
} elseif (!count($this->_lang_db)) {
$err = PEAR::raiseError('Language database has no elements.');
return false;
} else {
return true;
}
}
/**
* Omits languages
*
* Pass this function the name of or an array of names of
* languages that you don't want considered
*
* If you're only expecting a limited set of languages, this can greatly
* speed up processing
*
* @access public
* @param mixed $omit_list language name or array of names to omit
* @param bool $include_only if true will include (rather than
* exclude) only those in the list
* @return int number of languages successfully deleted
* @throws PEAR_Error
*/
function omitLanguages($omit_list, $include_only = false)
{
// setup check
if (!$this->_setup_ok($err)) {
return $err;
}
// deleting the given languages
if (!$include_only) {
if (!is_array($omit_list)) {
$omit_list = strtolower($omit_list); // case desensitize
if (isset($this->_lang_db[$omit_list])) {
unset($this->_lang_db[$omit_list]);
$deleted++;
}
} else {
foreach ($omit_list as $omit_lang) {
if (isset($this->_lang_db[$omit_lang])) {
unset($this->_lang_db[$omit_lang]);
$deleted++;
}
}
}
// deleting all except the given languages
} else {
if (!is_array($omit_list)) {
$omit_list = array($omit_list);
}
// case desensitize
foreach ($omit_list as $key => $omit_lang) {
$omit_list[$key] = strtolower($omit_lang);
}
foreach (array_keys($this->_lang_db) as $lang) {
if (!in_array($lang, $omit_list)) {
unset($this->_lang_db[$lang]);
$deleted++;
}
}
}
return $deleted;
}
/**
* Returns the number of languages that this object can detect
*
* @access public
* @return int the number of languages
* @throws PEAR_Error
*/
function getLanguageCount()
{
if (!$this->_setup_ok($err)) {
return $err;
} else {
return count($this->_lang_db);
}
}
/**
* Returns true if a given language exists
*
* If passed an array of names, will return true only if all exist
*
* @access public
* @param mixed $lang language name or array of language names
* @return bool true if language model exists
* @throws PEAR_Error
*/
function languageExists($lang)
{
if (!$this->_setup_ok($err)) {
return $err;
} else {
// string
if (is_string($lang)) {
return isset($this->_lang_db[strtolower($lang)]);
// array
} elseif (is_array($lang)) {
foreach ($lang as $test_lang) {
if (!isset($this->_lang_db[strtolower($test_lang)])) {
return false;
}
}
return true;
// other (error)
} else {
return PEAR::raiseError('Unknown type passed to languageExists()');
}
}
}
/**
* Returns the list of detectable languages
*
* @access public
* @return array the names of the languages known to this object
* @throws PEAR_Error
*/
function getLanguages()
{
if (!$this->_setup_ok($err)) {
return $err;
} else {
return array_keys($this->_lang_db);
}
}
/**
* Make this object behave like Language::Guess
*
* @access public
* @param bool $setting false to turn off perl compatibility
*/
function setPerlCompatible($setting = true)
{
if (is_bool($setting)) { // input check
$this->_perl_compatible = $setting;
if ($setting == true) {
$this->_max_score = $this->_threshold;
} else {
$this->_max_score = 0;
}
}
}
/**
* Converts a piece of text into trigrams
*
* @access private
* @param string $text text to convert
* @return array array of trigram frequencies
*/
function _trigram($text)
{
$text_length = strlen($text);
// input check
if ($text_length < 3) {
return array();
}
// each unique trigram is a key in this associative array
// number of times it appears in the string is the value
$trigram_freqs = array();
// $i keeps count of which byte in the string we're working in
// not which character, since characters could take from 1 - 4 bytes
$i = 0;
// $a, $b and $c each contain a single character
// with each iteration $b is set to $c, $a is set to $b and $c is set
// to the next character in $text
$a = $this->_next_char($text, $i, true);
$b = $this->_next_char($text, $i, true);
// starts off with the first two characters plus a space
if (!$this->_perl_compatible) {
if ($a != ' ') { // exclude trigrams with 2 contiguous spaces
$trigram_freqs[" $a$b"]++;
}
}
while ($i < $text_length) {
$c = $this->_next_char($text, $i, true);
// $i is incremented by reference in the line above
// exclude trigrams with 2 contiguous spaces
if (!($b == ' ' && ($a == ' ' || $c == ' '))) {
$trigram_freqs[$a . $b . $c]++;
}
$a = $b;
$b = $c;
}
// end with the last two characters plus a space
if ($b != ' ') { // exclude trigrams with 2 contiguous spaces
$trigram_freqs["$a$b "]++;
}
return $trigram_freqs;
}
/**
* Converts a set of trigrams from frequencies to ranks
*
* Thresholds (cuts off) the list at $this->_threshold
*
* @access private
* @param array $arr array of trgram
* @return array ranks of trigrams
*/
function _arr_rank(&$arr)
{
// sorts alphabetically first as a standard way of breaking rank ties
$this->_bub_sort($arr);
// below might also work, but seemed to introduce errors in testing
//ksort($arr);
//asort($arr);
$rank = array();
$i = 0;
foreach ($arr as $key => $value) {
$rank[$key] = $i++;
// cut off at a standard threshold
if ($i >= $this->_threshold) {
break;
}
}
return $rank;
}
/**
* Sorts an array by value breaking ties alphabetically
*
* @access private
* @param array &$arr the array to sort
*/
function _bub_sort(&$arr)
{
// should do the same as this perl statement:
// sort { $trigrams{$b} == $trigrams{$a} ? $a cmp $b : $trigrams{$b} <=> $trigrams{$a} }
// needs to sort by both key and value at once
// using the key to break ties for the value
// converts array into an array of arrays of each key and value
// may be a better way of doing this
$combined = array();
foreach ($arr as $key => $value) {
$combined[] = array($key, $value);
}
usort($combined, array($this, '_sort_func'));
$replacement = array();
foreach ($combined as $key => $value) {
list($new_key, $new_value) = $value;
$replacement[$new_key] = $new_value;
}
$arr = $replacement;
}
/**
* Sort function used by bubble sort
*
* Callback function for usort().
*
* @access private
* @param array first param passed by usort()
* @param array second param passed by usort()
* @return int 1 if $a is greater, -1 if not
* @see _bub_sort()
*/
function _sort_func($a, $b)
{
// each is actually a key/value pair, so that it can compare using both
list($a_key, $a_value) = $a;
list($b_key, $b_value) = $b;
// if the values are the same, break ties using the key
if ($a_value == $b_value) {
return strcmp($a_key, $b_key);
// if not, just sort normally
} else {
if ($a_value > $b_value) {
return -1;
} else {
return 1;
}
}
// 0 should not be possible because keys must be unique
}
/**
* Calculates a statistical difference between two sets of ranked trigrams
*
* Sums the differences in rank for each trigram. If the trigram does not
* appear in both, consider it a difference of $this->_threshold.
*
* @access private
* @param array $arr1 the reference set of trigram ranks
* @param array $arr2 the target set of trigram ranks
* @return int the sum of the differences between the ranks of
* the two trigram sets
*/
function _distance(&$arr1, &$arr2)
{
$sumdist = 0;
foreach ($arr2 as $key => $value) {
if (isset($arr1[$key])) {
$distance = abs($value - $arr1[$key]);
} else {
// $this->_threshold sets the maximum possible distance value
// for any one pair of trigrams
$distance = $this->_threshold;
}
$sumdist += $distance;
}
return $sumdist;
}
/**
* Normalizes the score returned by _distance()
*
* Different if perl compatible or not
*
* @access private
* @param int $score the score from _distance()
* @param int $base_count the number of trigrams being considered
* @return float the normalized score
* @see _distance()
*/
function _normalize_score($score, $base_count = null)
{
if ($base_count === null) {
$base_count = $this->_threshold;
}
if (!$this->_perl_compatible) {
return 1 - ($score / $base_count / $this->_threshold);
} else {
return floor($score / $base_count);
}
}
/**
* Detects the closeness of a sample of text to the known languages
*
* Calculates the statistical difference between the text and
* the trigrams for each language, normalizes the score then
* returns results for all languages in sorted order
*
* If perl compatible, the score is 300-0, 0 being most similar.
* Otherwise, it's 0-1 with 1 being most similar.
*
* The $sample text should be at least a few sentences in length;
* should be ascii-7 or utf8 encoded, if other and the mbstring extension
* is present it will try to detect and convert.
*
* @access public
* @param string $sample a sample of text to compare.
* @param int $limit if specified, return an array of the most likely
* $limit languages and their scores.
* @return mixed sorted array of language scores, blank array if no
* useable text was found, or PEAR_Error if error
* with the object setup
* @see _distance()
* @throws PEAR_Error
*/
function detect($sample, $limit = 0)
{
if (!$this->_setup_ok($err)) {
return $err;
}
// input check
if ($sample == '' || !preg_match('/\S/', $sample)) {
return array();
}
// check char encoding (only if mbstring extension is compiled)
if (function_exists('mb_detect_encoding') && function_exists('mb_convert_encoding')) {
$encoding = mb_detect_encoding($sample);
if ($encoding != 'ASCII' && $encoding != 'UTF-8') {
$sample = mb_convert_encoding($sample,'UTF-8');
}
}
$trigram_freqs = $this->_arr_rank($this->_trigram($sample));
$trigram_count = count($trigram_freqs);
if ($trigram_count == 0) {
return array();
}
// normalize the score
// by dividing it by the number of trigrams present
foreach ($this->_lang_db as $lang => $lang_arr) {
$scores[$lang] = $this->_normalize_score($this->_distance($lang_arr, $trigram_freqs), $trigram_count);
}
if ($this->_perl_compatible) {
asort($scores);
} else {
arsort($scores);
}
// todo: drop languages with a score of $this->_max_score?
// limit the number of returned scores
if ($limit && is_numeric($limit)) {
$limited_scores = array();
foreach ($scores as $key => $value) {
if ($i++ >= $limit) {
break;
}
$limited_scores[$key] = $value;
}
return $limited_scores;
} else {
return $scores;
}
}
/**
* Returns only the most similar language to the text sample
*
* Calls $this->detect() and returns only the top result
*
* @access public
* @param string $sample text to detect the language of
* @return string the name of the most likely language
* or null if no language is similar
* @see detect()
* @throws PEAR_Error
*/
function detectSimple($sample)
{
$scores = $this->detect($sample, 1);
if (PEAR::isError($scores)) {
return $scores;
}
// if top language has the maximum possible score,
// then the top score will have been picked at random
if (!is_array($scores) || !count($scores) || current($scores) == $this->_max_score) {
return null;
} else {
return ucfirst(key($scores));
}
}
/**
* Returns an array containing the most similar language and a confidence
* rating
*
* Confidence is a simple measure calculated from the similarity score
* minus the similarity score from the next most similar language
* divided by the highest possible score. Languages that have closely
* related cousins (e.g. Norwegian and Danish) should generally have lower
* confidence scores.
*
* The similarity score answers the question "How likely is the text the
* returned language regardless of the other languages considered?" The
* confidence score is one way of answering the question "how likely is the
* text the detected language relative to the rest of the language model
* set?"
*
* To see how similar languages are a priori, see languageSimilarity()
*
* @access public
* @param string $sample text for which language will be detected
* @return array most similar language, score and confidence rating
* or null if no language is similar
* @see detect()
* @throws PEAR_Error
*/
function detectConfidence($sample)
{
$scores = $this->detect($sample, 2);
if (PEAR::isError($scores)) {
return $scores;
}
// if most similar language has the max score, it
// will have been picked at random
if (!is_array($scores) || !count($scores) || current($scores) == $this->_max_score) {
return null;
}
$arr['language'] = ucfirst(key($scores));
$arr['similarity'] = current($scores);
if (next($scores) !== false) { // if false then no next element
// the goal is to return a higher value if the distance between
// the similarity of the first score and the second score is high
if ($this->_perl_compatible) {
$arr['confidence'] = (current($scores) - $arr['similarity']) / $this->_max_score;
} else {
$arr['confidence'] = $arr['similarity'] - current($scores);
}
} else {
$arr['confidence'] = null;
}
return $arr;
}
/**
* Calculate the similarities between the language models
*
* Use this function to see how similar languages are to each other.
*
* If passed 2 language names, will return just those languages compared.
* If passed 1 language name, will return that language compared to
* all others.
* If passed none, will return an array of every language model compared
* to every other one.
*
* @access public
* @param string $lang1 the name of the first language to be compared
* @param string $lang2 the name of the second language to be compared
* @return array scores of every language compared
* or the score of just the provided languages
* or null if one of the supplied languages does not exist
* @throws PEAR_Error
*/
function languageSimilarity($lang1 = null, $lang2 = null)
{
if (!$this->_setup_ok($err)) {
return $err;
}
if ($lang1 != null) {
$lang1 = strtolower($lang1);
// check if language model exists
if (!isset($this->_lang_db[$lang1])) {
return null;
}
if ($lang2 != null) {
// can't only set the second param
if ($lang1 == null) {
return null;
// check if language model exists
} elseif (!isset($this->_lang_db[$lang2])) {
return null;
}
$lang2 = strtolower($lang2);
// compare just these two languages
return $this->_normalize_score($this->_distance($this->_lang_db[$lang1], $this->_lang_db[$lang2]));
// compare just $lang1 to all languages
} else {
$return_arr = array();
foreach ($this->_lang_db as $key => $value) {
if ($key != $lang1) { // don't compare a language to itself
$return_arr[$key] = $this->_normalize_score($this->_distance($this->_lang_db[$lang1], $value));
}
}
asort($return_arr);
return $return_arr;
}
// compare all languages to each other
} else {
$return_arr = array();
foreach (array_keys($this->_lang_db) as $lang1) {
foreach (array_keys($this->_lang_db) as $lang2) {
if ($lang1 != $lang2) { // skip comparing languages to themselves
if (isset($return_arr[$lang2][$lang1])) {
$return_arr[$lang1][$lang2] = $return_arr[$lang2][$lang1];
} else {
$return_arr[$lang1][$lang2] = $this->_normalize_score($this->_distance($this->_lang_db[$lang1], $this->_lang_db[$lang2]));
}
}
}
}
return $return_arr;
}
}
/**
* utf8-safe fast character iterator
*
* Will get the next character starting from $counter, which will then be
* incremented. If a multi-byte char the bytes will be concatenated and
* $counter will be incremeted by the number of bytes in the char.
*
* @access private
* @param string &$str the string being iterated over
* @param int &$counter the iterator, will increment by reference
* @param bool $special_convert whether to do special conversions
* @return char the next (possibly multi-byte) char from $counter
*/
function _next_char(&$str, &$counter, $special_convert = false)
{
$char = $str{$counter++};
$ord = ord($char);
// for a description of the utf8 system see
// http://www.phpclasses.org/browse/file/5131.html
// normal ascii one byte char
if ($ord <= 127) {
// special conversions needed for this package
// (that only apply to regular ascii characters)
// lower case, and convert all non-alphanumeric characters
// other than "'" to space
if ($special_convert && $char != ' ' && $char != "'") {
if ($ord >= 65 && $ord <= 90) { // A-Z
$char = chr($ord + 32); // lower case
} elseif ($ord < 97 || $ord > 122) { // NOT a-z
$char = ' '; // convert to space
}
}
// multi-byte chars
} elseif ($ord >> 5 == 6) { // two-byte char
$nextchar = $str{$counter++}; // tag on next byte
// lower case latin accented characters
if ($special_convert && $ord == 195) {
$nextord = ord($nextchar);
$nextord_adj = $nextord + 64;
// for a reference, see
// http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html
if ($nextord_adj >= 192 && $nextord_adj <= 222 &&
$nextord_adj != 215) { // À - Þ but not ×
$nextchar = chr($nextord + 32); // lower case
}
}
$char .= $nextchar;
} elseif ($ord >> 4 == 14) { // three-byte char
// tag on next 2 bytes
$char .= $str{$counter++} . $str{$counter++};
} elseif ($ord >> 3 == 30) { // four-byte char
// tag on next 3 bytes
$char .= $str{$counter++} . $str{$counter++} . $str{$counter++};
} else {
// error?
}
return $char;
}
}
/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */
?>