<?php if (!defined('PmWiki')) exit();

/**

UTF-8 convertion & validation recipe

Strips any byte out of any given string that 
is not conforming to UTF-8 encoding. and plus
try to convert string to utf-8.

CHANGELOG:
눼눼눼눼눼
* changed the BOM value for utf8BOM var. (it was wrong)
* now if string is only 7-bit ASCII we don't do nothing,
  so you will be able to see the contents of PmWiki folder
  and create pages only in ASCII.
* removed function utf8_valid and replaced with utf8_compliant
  (smaller, faster) and fairly secure as we will remove 
  non-utf-8 chars with CharStrip Function.
* changed a bit (just for readability) the vars involved with MaxRuns
* included UTF-8 in $MbstringEncodings.
* included  SDVs for:
    $Utf8BOMInASCII - puts BOM even in pure ascii 7-bit
    $Utf8WithoutBOM - creates utf-8 without BOM
    $Utf8AlwaysRunConv - even if a string is pure ascii, run convertion to utf-8
  so other systems can benefit.
* implemeted a better BrowserEncoding

LICENSE:
눼눼눼눼
  Copyright 2001-2009 Carlos Bonamigo

  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 2 of the License, or
  (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

*/

// REGEX that strips utf-8 bad ranges 
// plus ASCII control characters
SDV($Utf8StripBad ,
'([\x09-\x0A\x0D\x20-\x7E]'.             # ASCII (excludes control chars)
'|[\xC2-\xDF][\x80-\xBF]'.               # non-overlong 2-byte
'|\xE0[\xA0-\xBF][\x80-\xBF]'.           # excluding overlongs
'|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'.    # straight 3-byte
'|\xED[\x80-\x9F][\x80-\xBF]'.           # excluding surrogates
'|\xF0[\x90-\xBF][\x80-\xBF]{2}'.        # planes 1-3
'|[\xF1-\xF3][\x80-\xBF]{3}'.            # planes 4-15
'|\xF4[\x80-\x8F][\x80-\xBF]{2}'.        # plane 16
'|(.{1}))');

/// FUNCTIONS

// Check if a string is 7-bit ASCII
function is_ascii($str) {
    // Search for any bytes which are outside the ASCII range...
    return (preg_match('/(?:[^\x00-\x7F])/',$str) !== 1);
}

// tests if a string is utf-8 compliant
// utf8_valid was replaced for not being
// as fast as this one. This will 
// pass some unicode chars that are
// not in the utf-8 range, but will be
// striped later on with CharStrip 
function utf8_compliant($str) {
    if ( strlen($str) == 0 ) {
        return TRUE;
    }
    // If even just the first character can be matched, when the /u
    // modifier is used, then it's valid UTF-8. If the UTF-8 is somehow
    // invalid, nothing at all will match, even if the string contains
    // some valid sequences
    return (preg_match('/^.{1}/us',$str,$ar) == 1);
}

// strips chars according to $ptr REGEX
function CharStrip($str,$ptr=''){
  global $Utf8StripBad;

  if($ptr == "") $ptr = $Utf8StripBad;

  ob_start();
  while (preg_match("/".$ptr."/S", $str, $matches)) {
      if ( !isset($matches[2])) {
          echo $matches[0];
      }
      $str = substr($str,strlen($matches[0]));
  }
  $result = ob_get_contents();
  ob_end_clean();
  return $result ;
}

function utf8conv($str,$bom='0'){
  global  $MbstringEncodings, $HebciEncoding,
          $Utf8ConvMaxRuns, $BrowserEncoding, $Charset;

  // 239 187 191 decinaml representation - old was wrong -> chr(255).chr(254);
  $utf8BOM = chr(239).chr(187).chr(191); 

  // find out browser accepted encodings
  if($BrowserEncoding != ''){
    $temp = explode(';',$_SERVER['HTTP_ACCEPT_CHARSET']);
    $temp = explode(',',$temp[0]);
    $BrowserEncoding = $temp; 
    $temp = '';
  }

  if($MbstringEncodings == "" && extension_loaded("mbstring")){

    $MbstringEncodings = array();
    $temp = mb_list_encodings(); $i = 1;

    foreach($temp as $k => $v){
      if($v == "pass" || $v == "auto" ) continue;
      else $MbstringEncodings[$i] = $v; $i++; //use $i=1 so array_search won't return FALSE if 0
    }

    $temp = ''; $i = '';
    mb_detect_order($MbstringEncodings);
  }

  SDV($Utf8BOMInASCII, 0);
  SDV($Utf8WithoutBOM, 0);
  SDV($Utf8AlwaysRunConv, 0);
  SDV($Utf8ConvMaxRuns, 7);

  if(is_ascii($str)){

    if($Utf8BOMInASCII == 1) $bom = 1;
    if($Utf8BOMInASCII == 0 && $bom == 1 ) $bom = 0;

    if($Utf8AlwaysRunConv == 1) $RunConv = 1;
    else $RunConv = 0;

  }elseif(utf8_compliant($str)){

    if($Utf8AlwaysRunConv == 1) $RunConv = 1; 
    else $RunConv = 0 ;

  }elseif(!utf8_compliant($str)){

    if($Utf8AlwaysRunConv == 0) $RunConv = 1 ;
    else $RunConv = 1 ;

  }

  $MaxRuns=1;

  while(!utf8_compliant($str) && $RunConv = 1) {

    if(extension_loaded('mbstring')){
      // find current encoding
      $strenc = mb_detect_encoding($str);
      // try some options to convert $str content to utf-8
      if( $strenc != '' ){
        if(array_search($strenc, $MbstringEncodings) != 0){
          $str = mb_convert_encoding($str,"UTF-8",$strenc);
        }
      }elseif( $strenc == '' && @$HebciEncoding != '' ){
        if(array_search($HebciEncoding, $MbstringEncodings) != 0){
          $str = mb_convert_encoding($str,"UTF-8",$HebciEncoding);
        }
      }
    }else{
      if($BrowserEncoding){
        if($BrowserEncoding[0] != '' 
             && $BrowserEncoding[0] != "UTF-8")
          $str = iconv($BrowserEncoding[0],"UTF-8",$str);
        if($BrowserEncoding[1] != ''
             && $BrowserEncoding[1] != "UTF-8")
          $str = iconv($BrowserEncoding[1],"UTF-8",$str);
      }elseif( $HebciEncoding != "" ){
        $str = iconv($HebciEncoding,"UTF-8",$str);
      }elseif( $Charset != "" ){
        $str = iconv($Charset,"UTF-8",$str);
      }else{
        $str = iconv("ISO-8859-1","UTF-8",$str);
      }
    }
    if($MaxRuns == $Utf8ConvMaxRuns) break;
    else $MaxRuns++;
  }

  $str = CharStrip($str);

  if($bom == 0 && substr($str,0,2) == $utf8BOM){
    // strip BOM
    while(substr($str,0,2) == $utf8BOM){
      $str = substr($str,3);
    }
  }elseif($bom == 1 && substr($str,0,2) != $utf8BOM ){
    // add BOM
    while(substr($str,0,2) == $utf8BOM){
      $str = substr($str,3);
    }
    $str= $utf8BOM.$str;
  }
  return $str;
}

// conversions

  // n
  if(@$_GET['n']) @$_GET['n'] = @utf8conv($_GET['n']);
  if(@$GLOBALS['n']) @$GLOBALS['n'] = @utf8conv($GLOBALS['n']);

  //pagename
  $pagename = utf8conv($pagename,0);
  $FmtPV['$RequestedPage'] = "'".htmlspecialchars($pagename, ENT_QUOTES)."'";
  $Cursor['*'] = &$pagename;

  // if editing convert text so we always save utf-8
  if(@$_POST['text']) @$_POST['text'] = @utf8conv($_POST['text']);