mirror of
https://git.lyx.org/repos/lyx.git
synced 2024-11-09 18:31:04 +00:00
Do not use of boost::tokenizer
It is easier to use instead getVectorFromString for the use we have of this tokenizer. The two places are environment.cpp (path stuff) and qt_helpers (file fileters). The new code is much shorter. This allow to remove boost/tokenizer.hpp and friends from our boost tree.
This commit is contained in:
Jean-Marc Lasgouttes
Richard Heck
parent
87d4ce0ff7
commit
68a1f9a44e
651
3rdparty/boost/boost/token_functions.hpp
vendored
651
3rdparty/boost/boost/token_functions.hpp
vendored
@ -1,651 +0,0 @@
|
||||
// Boost token_functions.hpp ------------------------------------------------//
|
||||
|
||||
// Copyright John R. Bandela 2001.
|
||||
|
||||
// Distributed under the Boost Software License, Version 1.0. (See
|
||||
// accompanying file LICENSE_1_0.txt or copy at
|
||||
// http://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
// See http://www.boost.org/libs/tokenizer/ for documentation.
|
||||
|
||||
// Revision History:
|
||||
// 01 Oct 2004 Joaquin M Lopez Munoz
|
||||
// Workaround for a problem with string::assign in msvc-stlport
|
||||
// 06 Apr 2004 John Bandela
|
||||
// Fixed a bug involving using char_delimiter with a true input iterator
|
||||
// 28 Nov 2003 Robert Zeh and John Bandela
|
||||
// Converted into "fast" functions that avoid using += when
|
||||
// the supplied iterator isn't an input_iterator; based on
|
||||
// some work done at Archelon and a version that was checked into
|
||||
// the boost CVS for a short period of time.
|
||||
// 20 Feb 2002 John Maddock
|
||||
// Removed using namespace std declarations and added
|
||||
// workaround for BOOST_NO_STDC_NAMESPACE (the library
|
||||
// can be safely mixed with regex).
|
||||
// 06 Feb 2002 Jeremy Siek
|
||||
// Added char_separator.
|
||||
// 02 Feb 2002 Jeremy Siek
|
||||
// Removed tabs and a little cleanup.
|
||||
|
||||
|
||||
#ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
|
||||
#define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
|
||||
|
||||
#include <vector>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <cctype>
|
||||
#include <algorithm> // for find_if
|
||||
#include <boost/config.hpp>
|
||||
#include <boost/assert.hpp>
|
||||
#include <boost/detail/workaround.hpp>
|
||||
#include <boost/mpl/if.hpp>
|
||||
#include <boost/throw_exception.hpp>
|
||||
#if !defined(BOOST_NO_CWCTYPE)
|
||||
#include <cwctype>
|
||||
#endif
|
||||
|
||||
//
|
||||
// the following must not be macros if we are to prefix them
|
||||
// with std:: (they shouldn't be macros anyway...)
|
||||
//
|
||||
#ifdef ispunct
|
||||
# undef ispunct
|
||||
#endif
|
||||
#ifdef iswpunct
|
||||
# undef iswpunct
|
||||
#endif
|
||||
#ifdef isspace
|
||||
# undef isspace
|
||||
#endif
|
||||
#ifdef iswspace
|
||||
# undef iswspace
|
||||
#endif
|
||||
//
|
||||
// fix namespace problems:
|
||||
//
|
||||
#ifdef BOOST_NO_STDC_NAMESPACE
|
||||
namespace std{
|
||||
using ::ispunct;
|
||||
using ::isspace;
|
||||
#if !defined(BOOST_NO_CWCTYPE)
|
||||
using ::iswpunct;
|
||||
using ::iswspace;
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
namespace boost{
|
||||
//===========================================================================
|
||||
// The escaped_list_separator class. Which is a model of TokenizerFunction
|
||||
// An escaped list is a super-set of what is commonly known as a comma
|
||||
// separated value (csv) list.It is separated into fields by a comma or
|
||||
// other character. If the delimiting character is inside quotes, then it is
|
||||
// counted as a regular character.To allow for embedded quotes in a field,
|
||||
// there can be escape sequences using the \ much like C.
|
||||
// The role of the comma, the quotation mark, and the escape
|
||||
// character (backslash \), can be assigned to other characters.
|
||||
|
||||
struct escaped_list_error : public std::runtime_error{
|
||||
escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
|
||||
};
|
||||
|
||||
|
||||
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
|
||||
// MSVC does not like the following typename
|
||||
template <class Char,
|
||||
class Traits = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
|
||||
class escaped_list_separator {
|
||||
|
||||
private:
|
||||
typedef std::basic_string<Char,Traits> string_type;
|
||||
struct char_eq {
|
||||
Char e_;
|
||||
char_eq(Char e):e_(e) { }
|
||||
bool operator()(Char c) {
|
||||
return Traits::eq(e_,c);
|
||||
}
|
||||
};
|
||||
string_type escape_;
|
||||
string_type c_;
|
||||
string_type quote_;
|
||||
bool last_;
|
||||
|
||||
bool is_escape(Char e) {
|
||||
char_eq f(e);
|
||||
return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
|
||||
}
|
||||
bool is_c(Char e) {
|
||||
char_eq f(e);
|
||||
return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
|
||||
}
|
||||
bool is_quote(Char e) {
|
||||
char_eq f(e);
|
||||
return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
|
||||
}
|
||||
template <typename iterator, typename Token>
|
||||
void do_escape(iterator& next,iterator end,Token& tok) {
|
||||
if (++next == end)
|
||||
BOOST_THROW_EXCEPTION(escaped_list_error(std::string("cannot end with escape")));
|
||||
if (Traits::eq(*next,'n')) {
|
||||
tok+='\n';
|
||||
return;
|
||||
}
|
||||
else if (is_quote(*next)) {
|
||||
tok+=*next;
|
||||
return;
|
||||
}
|
||||
else if (is_c(*next)) {
|
||||
tok+=*next;
|
||||
return;
|
||||
}
|
||||
else if (is_escape(*next)) {
|
||||
tok+=*next;
|
||||
return;
|
||||
}
|
||||
else
|
||||
BOOST_THROW_EXCEPTION(escaped_list_error(std::string("unknown escape sequence")));
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
explicit escaped_list_separator(Char e = '\\',
|
||||
Char c = ',',Char q = '\"')
|
||||
: escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
|
||||
|
||||
escaped_list_separator(string_type e, string_type c, string_type q)
|
||||
: escape_(e), c_(c), quote_(q), last_(false) { }
|
||||
|
||||
void reset() {last_=false;}
|
||||
|
||||
template <typename InputIterator, typename Token>
|
||||
bool operator()(InputIterator& next,InputIterator end,Token& tok) {
|
||||
bool bInQuote = false;
|
||||
tok = Token();
|
||||
|
||||
if (next == end) {
|
||||
if (last_) {
|
||||
last_ = false;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
}
|
||||
last_ = false;
|
||||
for (;next != end;++next) {
|
||||
if (is_escape(*next)) {
|
||||
do_escape(next,end,tok);
|
||||
}
|
||||
else if (is_c(*next)) {
|
||||
if (!bInQuote) {
|
||||
// If we are not in quote, then we are done
|
||||
++next;
|
||||
// The last character was a c, that means there is
|
||||
// 1 more blank field
|
||||
last_ = true;
|
||||
return true;
|
||||
}
|
||||
else tok+=*next;
|
||||
}
|
||||
else if (is_quote(*next)) {
|
||||
bInQuote=!bInQuote;
|
||||
}
|
||||
else {
|
||||
tok += *next;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
//===========================================================================
|
||||
// The classes here are used by offset_separator and char_separator to implement
|
||||
// faster assigning of tokens using assign instead of +=
|
||||
|
||||
namespace tokenizer_detail {
|
||||
//===========================================================================
|
||||
// Tokenizer was broken for wide character separators, at least on Windows, since
|
||||
// CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts
|
||||
// if higher values are passed in. The traits extension class should take care of this.
|
||||
// Assuming that the conditional will always get optimized out in the function
|
||||
// implementations, argument types are not a problem since both forms of character classifiers
|
||||
// expect an int.
|
||||
|
||||
#if !defined(BOOST_NO_CWCTYPE)
|
||||
template<typename traits, int N>
|
||||
struct traits_extension_details : public traits {
|
||||
typedef typename traits::char_type char_type;
|
||||
static bool isspace(char_type c)
|
||||
{
|
||||
return std::iswspace(c) != 0;
|
||||
}
|
||||
static bool ispunct(char_type c)
|
||||
{
|
||||
return std::iswpunct(c) != 0;
|
||||
}
|
||||
};
|
||||
|
||||
template<typename traits>
|
||||
struct traits_extension_details<traits, 1> : public traits {
|
||||
typedef typename traits::char_type char_type;
|
||||
static bool isspace(char_type c)
|
||||
{
|
||||
return std::isspace(c) != 0;
|
||||
}
|
||||
static bool ispunct(char_type c)
|
||||
{
|
||||
return std::ispunct(c) != 0;
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
|
||||
// In case there is no cwctype header, we implement the checks manually.
|
||||
// We make use of the fact that the tested categories should fit in ASCII.
|
||||
template<typename traits>
|
||||
struct traits_extension : public traits {
|
||||
typedef typename traits::char_type char_type;
|
||||
static bool isspace(char_type c)
|
||||
{
|
||||
#if !defined(BOOST_NO_CWCTYPE)
|
||||
return traits_extension_details<traits, sizeof(char_type)>::isspace(c);
|
||||
#else
|
||||
return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool ispunct(char_type c)
|
||||
{
|
||||
#if !defined(BOOST_NO_CWCTYPE)
|
||||
return traits_extension_details<traits, sizeof(char_type)>::ispunct(c);
|
||||
#else
|
||||
return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
// The assign_or_plus_equal struct contains functions that implement
|
||||
// assign, +=, and clearing based on the iterator type. The
|
||||
// generic case does nothing for plus_equal and clearing, while
|
||||
// passing through the call for assign.
|
||||
//
|
||||
// When an input iterator is being used, the situation is reversed.
|
||||
// The assign method does nothing, plus_equal invokes operator +=,
|
||||
// and the clearing method sets the supplied token to the default
|
||||
// token constructor's result.
|
||||
//
|
||||
|
||||
template<class IteratorTag>
|
||||
struct assign_or_plus_equal {
|
||||
template<class Iterator, class Token>
|
||||
static void assign(Iterator b, Iterator e, Token &t) {
|
||||
t.assign(b, e);
|
||||
}
|
||||
|
||||
template<class Token, class Value>
|
||||
static void plus_equal(Token &, const Value &) { }
|
||||
|
||||
// If we are doing an assign, there is no need for the
|
||||
// the clear.
|
||||
//
|
||||
template<class Token>
|
||||
static void clear(Token &) { }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct assign_or_plus_equal<std::input_iterator_tag> {
|
||||
template<class Iterator, class Token>
|
||||
static void assign(Iterator , Iterator , Token &) { }
|
||||
template<class Token, class Value>
|
||||
static void plus_equal(Token &t, const Value &v) {
|
||||
t += v;
|
||||
}
|
||||
template<class Token>
|
||||
static void clear(Token &t) {
|
||||
t = Token();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template<class Iterator>
|
||||
struct pointer_iterator_category{
|
||||
typedef std::random_access_iterator_tag type;
|
||||
};
|
||||
|
||||
|
||||
template<class Iterator>
|
||||
struct class_iterator_category{
|
||||
typedef typename Iterator::iterator_category type;
|
||||
};
|
||||
|
||||
|
||||
|
||||
// This portably gets the iterator_tag without partial template specialization
|
||||
template<class Iterator>
|
||||
struct get_iterator_category{
|
||||
typedef typename mpl::if_<is_pointer<Iterator>,
|
||||
pointer_iterator_category<Iterator>,
|
||||
class_iterator_category<Iterator>
|
||||
>::type cat;
|
||||
|
||||
typedef typename cat::type iterator_category;
|
||||
};
|
||||
|
||||
|
||||
} // namespace tokenizer_detail
|
||||
|
||||
|
||||
//===========================================================================
|
||||
// The offset_separator class, which is a model of TokenizerFunction.
|
||||
// Offset breaks a string into tokens based on a range of offsets
|
||||
|
||||
class offset_separator {
|
||||
private:
|
||||
|
||||
std::vector<int> offsets_;
|
||||
unsigned int current_offset_;
|
||||
bool wrap_offsets_;
|
||||
bool return_partial_last_;
|
||||
|
||||
public:
|
||||
template <typename Iter>
|
||||
offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
|
||||
bool return_partial_last = true)
|
||||
: offsets_(begin,end), current_offset_(0),
|
||||
wrap_offsets_(wrap_offsets),
|
||||
return_partial_last_(return_partial_last) { }
|
||||
|
||||
offset_separator()
|
||||
: offsets_(1,1), current_offset_(),
|
||||
wrap_offsets_(true), return_partial_last_(true) { }
|
||||
|
||||
void reset() {
|
||||
current_offset_ = 0;
|
||||
}
|
||||
|
||||
template <typename InputIterator, typename Token>
|
||||
bool operator()(InputIterator& next, InputIterator end, Token& tok)
|
||||
{
|
||||
typedef tokenizer_detail::assign_or_plus_equal<
|
||||
BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
|
||||
InputIterator
|
||||
>::iterator_category
|
||||
> assigner;
|
||||
|
||||
BOOST_ASSERT(!offsets_.empty());
|
||||
|
||||
assigner::clear(tok);
|
||||
InputIterator start(next);
|
||||
|
||||
if (next == end)
|
||||
return false;
|
||||
|
||||
if (current_offset_ == offsets_.size())
|
||||
{
|
||||
if (wrap_offsets_)
|
||||
current_offset_=0;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
int c = offsets_[current_offset_];
|
||||
int i = 0;
|
||||
for (; i < c; ++i) {
|
||||
if (next == end)break;
|
||||
assigner::plus_equal(tok,*next++);
|
||||
}
|
||||
assigner::assign(start,next,tok);
|
||||
|
||||
if (!return_partial_last_)
|
||||
if (i < (c-1) )
|
||||
return false;
|
||||
|
||||
++current_offset_;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
//===========================================================================
|
||||
// The char_separator class breaks a sequence of characters into
|
||||
// tokens based on the character delimiters (very much like bad old
|
||||
// strtok). A delimiter character can either be kept or dropped. A
|
||||
// kept delimiter shows up as an output token, whereas a dropped
|
||||
// delimiter does not.
|
||||
|
||||
// This class replaces the char_delimiters_separator class. The
|
||||
// constructor for the char_delimiters_separator class was too
|
||||
// confusing and needed to be deprecated. However, because of the
|
||||
// default arguments to the constructor, adding the new constructor
|
||||
// would cause ambiguity, so instead I deprecated the whole class.
|
||||
// The implementation of the class was also simplified considerably.
|
||||
|
||||
enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
|
||||
|
||||
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
|
||||
template <typename Char,
|
||||
typename Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
|
||||
class char_separator
|
||||
{
|
||||
typedef tokenizer_detail::traits_extension<Tr> Traits;
|
||||
typedef std::basic_string<Char,Tr> string_type;
|
||||
public:
|
||||
explicit
|
||||
char_separator(const Char* dropped_delims,
|
||||
const Char* kept_delims = 0,
|
||||
empty_token_policy empty_tokens = drop_empty_tokens)
|
||||
: m_dropped_delims(dropped_delims),
|
||||
m_use_ispunct(false),
|
||||
m_use_isspace(false),
|
||||
m_empty_tokens(empty_tokens),
|
||||
m_output_done(false)
|
||||
{
|
||||
// Borland workaround
|
||||
if (kept_delims)
|
||||
m_kept_delims = kept_delims;
|
||||
}
|
||||
|
||||
// use ispunct() for kept delimiters and isspace for dropped.
|
||||
explicit
|
||||
char_separator()
|
||||
: m_use_ispunct(true),
|
||||
m_use_isspace(true),
|
||||
m_empty_tokens(drop_empty_tokens) { }
|
||||
|
||||
void reset() { }
|
||||
|
||||
template <typename InputIterator, typename Token>
|
||||
bool operator()(InputIterator& next, InputIterator end, Token& tok)
|
||||
{
|
||||
typedef tokenizer_detail::assign_or_plus_equal<
|
||||
BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
|
||||
InputIterator
|
||||
>::iterator_category
|
||||
> assigner;
|
||||
|
||||
assigner::clear(tok);
|
||||
|
||||
// skip past all dropped_delims
|
||||
if (m_empty_tokens == drop_empty_tokens)
|
||||
for (; next != end && is_dropped(*next); ++next)
|
||||
{ }
|
||||
|
||||
InputIterator start(next);
|
||||
|
||||
if (m_empty_tokens == drop_empty_tokens) {
|
||||
|
||||
if (next == end)
|
||||
return false;
|
||||
|
||||
|
||||
// if we are on a kept_delims move past it and stop
|
||||
if (is_kept(*next)) {
|
||||
assigner::plus_equal(tok,*next);
|
||||
++next;
|
||||
} else
|
||||
// append all the non delim characters
|
||||
for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
|
||||
assigner::plus_equal(tok,*next);
|
||||
}
|
||||
else { // m_empty_tokens == keep_empty_tokens
|
||||
|
||||
// Handle empty token at the end
|
||||
if (next == end)
|
||||
{
|
||||
if (m_output_done == false)
|
||||
{
|
||||
m_output_done = true;
|
||||
assigner::assign(start,next,tok);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
if (is_kept(*next)) {
|
||||
if (m_output_done == false)
|
||||
m_output_done = true;
|
||||
else {
|
||||
assigner::plus_equal(tok,*next);
|
||||
++next;
|
||||
m_output_done = false;
|
||||
}
|
||||
}
|
||||
else if (m_output_done == false && is_dropped(*next)) {
|
||||
m_output_done = true;
|
||||
}
|
||||
else {
|
||||
if (is_dropped(*next))
|
||||
start=++next;
|
||||
for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
|
||||
assigner::plus_equal(tok,*next);
|
||||
m_output_done = true;
|
||||
}
|
||||
}
|
||||
assigner::assign(start,next,tok);
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
string_type m_kept_delims;
|
||||
string_type m_dropped_delims;
|
||||
bool m_use_ispunct;
|
||||
bool m_use_isspace;
|
||||
empty_token_policy m_empty_tokens;
|
||||
bool m_output_done;
|
||||
|
||||
bool is_kept(Char E) const
|
||||
{
|
||||
if (m_kept_delims.length())
|
||||
return m_kept_delims.find(E) != string_type::npos;
|
||||
else if (m_use_ispunct) {
|
||||
return Traits::ispunct(E) != 0;
|
||||
} else
|
||||
return false;
|
||||
}
|
||||
bool is_dropped(Char E) const
|
||||
{
|
||||
if (m_dropped_delims.length())
|
||||
return m_dropped_delims.find(E) != string_type::npos;
|
||||
else if (m_use_isspace) {
|
||||
return Traits::isspace(E) != 0;
|
||||
} else
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//===========================================================================
|
||||
// The following class is DEPRECATED, use class char_separators instead.
|
||||
//
|
||||
// The char_delimiters_separator class, which is a model of
|
||||
// TokenizerFunction. char_delimiters_separator breaks a string
|
||||
// into tokens based on character delimiters. There are 2 types of
|
||||
// delimiters. returnable delimiters can be returned as
|
||||
// tokens. These are often punctuation. nonreturnable delimiters
|
||||
// cannot be returned as tokens. These are often whitespace
|
||||
|
||||
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
|
||||
template <class Char,
|
||||
class Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
|
||||
class char_delimiters_separator {
|
||||
private:
|
||||
|
||||
typedef tokenizer_detail::traits_extension<Tr> Traits;
|
||||
typedef std::basic_string<Char,Tr> string_type;
|
||||
string_type returnable_;
|
||||
string_type nonreturnable_;
|
||||
bool return_delims_;
|
||||
bool no_ispunct_;
|
||||
bool no_isspace_;
|
||||
|
||||
bool is_ret(Char E)const
|
||||
{
|
||||
if (returnable_.length())
|
||||
return returnable_.find(E) != string_type::npos;
|
||||
else{
|
||||
if (no_ispunct_) {return false;}
|
||||
else{
|
||||
int r = Traits::ispunct(E);
|
||||
return r != 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
bool is_nonret(Char E)const
|
||||
{
|
||||
if (nonreturnable_.length())
|
||||
return nonreturnable_.find(E) != string_type::npos;
|
||||
else{
|
||||
if (no_isspace_) {return false;}
|
||||
else{
|
||||
int r = Traits::isspace(E);
|
||||
return r != 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
explicit char_delimiters_separator(bool return_delims = false,
|
||||
const Char* returnable = 0,
|
||||
const Char* nonreturnable = 0)
|
||||
: returnable_(returnable ? returnable : string_type().c_str()),
|
||||
nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
|
||||
return_delims_(return_delims), no_ispunct_(returnable!=0),
|
||||
no_isspace_(nonreturnable!=0) { }
|
||||
|
||||
void reset() { }
|
||||
|
||||
public:
|
||||
|
||||
template <typename InputIterator, typename Token>
|
||||
bool operator()(InputIterator& next, InputIterator end,Token& tok) {
|
||||
tok = Token();
|
||||
|
||||
// skip past all nonreturnable delims
|
||||
// skip past the returnable only if we are not returning delims
|
||||
for (;next!=end && ( is_nonret(*next) || (is_ret(*next)
|
||||
&& !return_delims_ ) );++next) { }
|
||||
|
||||
if (next == end) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// if we are to return delims and we are one a returnable one
|
||||
// move past it and stop
|
||||
if (is_ret(*next) && return_delims_) {
|
||||
tok+=*next;
|
||||
++next;
|
||||
}
|
||||
else
|
||||
// append all the non delim characters
|
||||
for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)
|
||||
tok+=*next;
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
} //namespace boost
|
||||
|
||||
#endif
|
||||
128
3rdparty/boost/boost/token_iterator.hpp
vendored
128
3rdparty/boost/boost/token_iterator.hpp
vendored
@ -1,128 +0,0 @@
|
||||
// Boost token_iterator.hpp -------------------------------------------------//
|
||||
|
||||
// Copyright John R. Bandela 2001
|
||||
// Distributed under the Boost Software License, Version 1.0. (See
|
||||
// accompanying file LICENSE_1_0.txt or copy at
|
||||
// http://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
// See http://www.boost.org/libs/tokenizer for documentation.
|
||||
|
||||
// Revision History:
|
||||
// 16 Jul 2003 John Bandela
|
||||
// Allowed conversions from convertible base iterators
|
||||
// 03 Jul 2003 John Bandela
|
||||
// Converted to new iterator adapter
|
||||
|
||||
|
||||
|
||||
#ifndef BOOST_TOKENIZER_POLICY_JRB070303_HPP_
|
||||
#define BOOST_TOKENIZER_POLICY_JRB070303_HPP_
|
||||
|
||||
#include <boost/assert.hpp>
|
||||
#include <boost/iterator/iterator_adaptor.hpp>
|
||||
#include <boost/iterator/minimum_category.hpp>
|
||||
#include <boost/token_functions.hpp>
|
||||
#include <utility>
|
||||
|
||||
namespace boost
|
||||
{
|
||||
template <class TokenizerFunc, class Iterator, class Type>
|
||||
class token_iterator
|
||||
: public iterator_facade<
|
||||
token_iterator<TokenizerFunc, Iterator, Type>
|
||||
, Type
|
||||
, typename iterators::minimum_category<
|
||||
forward_traversal_tag
|
||||
, typename iterator_traversal<Iterator>::type
|
||||
>::type
|
||||
, const Type&
|
||||
>
|
||||
{
|
||||
|
||||
friend class iterator_core_access;
|
||||
|
||||
TokenizerFunc f_;
|
||||
Iterator begin_;
|
||||
Iterator end_;
|
||||
bool valid_;
|
||||
Type tok_;
|
||||
|
||||
void increment(){
|
||||
BOOST_ASSERT(valid_);
|
||||
valid_ = f_(begin_,end_,tok_);
|
||||
}
|
||||
|
||||
const Type& dereference() const {
|
||||
BOOST_ASSERT(valid_);
|
||||
return tok_;
|
||||
}
|
||||
template<class Other>
|
||||
bool equal(const Other& a) const{
|
||||
return (a.valid_ && valid_)
|
||||
?( (a.begin_==begin_) && (a.end_ == end_) )
|
||||
:(a.valid_==valid_);
|
||||
|
||||
}
|
||||
|
||||
void initialize(){
|
||||
if(valid_) return;
|
||||
f_.reset();
|
||||
valid_ = (begin_ != end_)?
|
||||
f_(begin_,end_,tok_):false;
|
||||
}
|
||||
public:
|
||||
token_iterator():begin_(),end_(),valid_(false),tok_() { }
|
||||
|
||||
token_iterator(TokenizerFunc f, Iterator begin, Iterator e = Iterator())
|
||||
: f_(f),begin_(begin),end_(e),valid_(false),tok_(){ initialize(); }
|
||||
|
||||
token_iterator(Iterator begin, Iterator e = Iterator())
|
||||
: f_(),begin_(begin),end_(e),valid_(false),tok_() {initialize();}
|
||||
|
||||
template<class OtherIter>
|
||||
token_iterator(
|
||||
token_iterator<TokenizerFunc, OtherIter,Type> const& t
|
||||
, typename enable_if_convertible<OtherIter, Iterator>::type* = 0)
|
||||
: f_(t.tokenizer_function()),begin_(t.base())
|
||||
,end_(t.end()),valid_(!t.at_end()),tok_(t.current_token()) {}
|
||||
|
||||
Iterator base()const{return begin_;}
|
||||
|
||||
Iterator end()const{return end_;}
|
||||
|
||||
TokenizerFunc tokenizer_function()const{return f_;}
|
||||
|
||||
Type current_token()const{return tok_;}
|
||||
|
||||
bool at_end()const{return !valid_;}
|
||||
|
||||
|
||||
|
||||
|
||||
};
|
||||
template <
|
||||
class TokenizerFunc = char_delimiters_separator<char>,
|
||||
class Iterator = std::string::const_iterator,
|
||||
class Type = std::string
|
||||
>
|
||||
class token_iterator_generator {
|
||||
|
||||
private:
|
||||
public:
|
||||
typedef token_iterator<TokenizerFunc,Iterator,Type> type;
|
||||
};
|
||||
|
||||
|
||||
// Type has to be first because it needs to be explicitly specified
|
||||
// because there is no way the function can deduce it.
|
||||
template<class Type, class Iterator, class TokenizerFunc>
|
||||
typename token_iterator_generator<TokenizerFunc,Iterator,Type>::type
|
||||
make_token_iterator(Iterator begin, Iterator end,const TokenizerFunc& fun){
|
||||
typedef typename
|
||||
token_iterator_generator<TokenizerFunc,Iterator,Type>::type ret_type;
|
||||
return ret_type(fun,begin,end);
|
||||
}
|
||||
|
||||
} // namespace boost
|
||||
|
||||
#endif
|
||||
98
3rdparty/boost/boost/tokenizer.hpp
vendored
98
3rdparty/boost/boost/tokenizer.hpp
vendored
@ -1,98 +0,0 @@
|
||||
// Boost tokenizer.hpp -----------------------------------------------------//
|
||||
|
||||
// (c) Copyright Jeremy Siek and John R. Bandela 2001.
|
||||
|
||||
// Distributed under the Boost Software License, Version 1.0. (See
|
||||
// accompanying file LICENSE_1_0.txt or copy at
|
||||
// http://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
// See http://www.boost.org/libs/tokenizer for documenation
|
||||
|
||||
// Revision History:
|
||||
// 03 Jul 2003 John Bandela
|
||||
// Converted to new iterator adapter
|
||||
// 02 Feb 2002 Jeremy Siek
|
||||
// Removed tabs and a little cleanup.
|
||||
|
||||
#ifndef BOOST_TOKENIZER_JRB070303_HPP_
|
||||
#define BOOST_TOKENIZER_JRB070303_HPP_
|
||||
|
||||
#include <boost/token_iterator.hpp>
|
||||
|
||||
namespace boost {
|
||||
|
||||
|
||||
//===========================================================================
|
||||
// A container-view of a tokenized "sequence"
|
||||
template <
|
||||
typename TokenizerFunc = char_delimiters_separator<char>,
|
||||
typename Iterator = std::string::const_iterator,
|
||||
typename Type = std::string
|
||||
>
|
||||
class tokenizer {
|
||||
private:
|
||||
typedef token_iterator_generator<TokenizerFunc,Iterator,Type> TGen;
|
||||
|
||||
// It seems that MSVC does not like the unqualified use of iterator,
|
||||
// Thus we use iter internally when it is used unqualified and
|
||||
// the users of this class will always qualify iterator.
|
||||
typedef typename TGen::type iter;
|
||||
|
||||
public:
|
||||
|
||||
typedef iter iterator;
|
||||
typedef iter const_iterator;
|
||||
typedef Type value_type;
|
||||
typedef value_type& reference;
|
||||
typedef const value_type& const_reference;
|
||||
typedef value_type* pointer;
|
||||
typedef const pointer const_pointer;
|
||||
typedef void size_type;
|
||||
typedef void difference_type;
|
||||
|
||||
tokenizer(Iterator first, Iterator last,
|
||||
const TokenizerFunc& f = TokenizerFunc())
|
||||
: first_(first), last_(last), f_(f) { }
|
||||
|
||||
template <typename Container>
|
||||
tokenizer(const Container& c)
|
||||
: first_(c.begin()), last_(c.end()), f_() { }
|
||||
|
||||
template <typename Container>
|
||||
tokenizer(const Container& c,const TokenizerFunc& f)
|
||||
: first_(c.begin()), last_(c.end()), f_(f) { }
|
||||
|
||||
void assign(Iterator first, Iterator last){
|
||||
first_ = first;
|
||||
last_ = last;
|
||||
}
|
||||
|
||||
void assign(Iterator first, Iterator last, const TokenizerFunc& f){
|
||||
assign(first,last);
|
||||
f_ = f;
|
||||
}
|
||||
|
||||
template <typename Container>
|
||||
void assign(const Container& c){
|
||||
assign(c.begin(),c.end());
|
||||
}
|
||||
|
||||
|
||||
template <typename Container>
|
||||
void assign(const Container& c, const TokenizerFunc& f){
|
||||
assign(c.begin(),c.end(),f);
|
||||
}
|
||||
|
||||
iter begin() const { return iter(f_,first_,last_); }
|
||||
iter end() const { return iter(f_,last_,last_); }
|
||||
|
||||
private:
|
||||
Iterator first_;
|
||||
Iterator last_;
|
||||
TokenizerFunc f_;
|
||||
};
|
||||
|
||||
|
||||
} // namespace boost
|
||||
|
||||
#endif
|
||||
1
3rdparty/boost/extract.sh
vendored
1
3rdparty/boost/extract.sh
vendored
@ -36,7 +36,6 @@ bcp --boost=$1 \
|
||||
boost/signal.hpp \
|
||||
boost/signals/connection.hpp \
|
||||
boost/signals/trackable.hpp \
|
||||
boost/tokenizer.hpp \
|
||||
boost/tuple/tuple.hpp \
|
||||
boost/mpl/string.hpp \
|
||||
boost/mpl/fold.hpp \
|
||||
|
||||
@ -51,7 +51,6 @@
|
||||
// for FileFilter.
|
||||
// FIXME: Remove
|
||||
#include "support/regex.h"
|
||||
#include <boost/tokenizer.hpp>
|
||||
|
||||
|
||||
using namespace std;
|
||||
@ -464,16 +463,12 @@ struct Filter
|
||||
Filter::Filter(docstring const & description, string const & globs)
|
||||
: desc_(description)
|
||||
{
|
||||
typedef boost::tokenizer<boost::char_separator<char> > Tokenizer;
|
||||
boost::char_separator<char> const separator(" ");
|
||||
|
||||
// Given "<glob> <glob> ... *.{abc,def} <glob>", expand to
|
||||
// "<glob> <glob> ... *.abc *.def <glob>"
|
||||
string const expanded_globs = convert_brace_glob(globs);
|
||||
|
||||
// Split into individual globs.
|
||||
Tokenizer const tokens(expanded_globs, separator);
|
||||
globs_ = vector<string>(tokens.begin(), tokens.end());
|
||||
globs_ = getVectorFromString(expanded_globs, " ");
|
||||
}
|
||||
|
||||
|
||||
@ -488,11 +483,7 @@ QString Filter::toString() const
|
||||
s += " (";
|
||||
}
|
||||
|
||||
for (size_t i = 0; i != globs_.size(); ++i) {
|
||||
if (i > 0)
|
||||
s += ' ';
|
||||
s += toqstr(globs_[i]);
|
||||
}
|
||||
s += toqstr(getStringFromVector(globs_, " "));
|
||||
|
||||
if (has_description)
|
||||
s += ')';
|
||||
|
||||
@ -15,10 +15,10 @@
|
||||
#include "support/environment.h"
|
||||
|
||||
#include "support/docstring.h"
|
||||
#include "support/lstrings.h"
|
||||
#include "support/os.h"
|
||||
|
||||
#include <boost/tokenizer.hpp>
|
||||
|
||||
#include <algorithm> // for remove
|
||||
#include <cstdlib>
|
||||
#include <map>
|
||||
#include <sstream>
|
||||
@ -45,20 +45,10 @@ string const getEnv(string const & name)
|
||||
|
||||
vector<string> const getEnvPath(string const & name)
|
||||
{
|
||||
typedef boost::char_separator<char> Separator;
|
||||
typedef boost::tokenizer<Separator> Tokenizer;
|
||||
|
||||
string const env_var = getEnv(name);
|
||||
Separator const separator(string(1, os::path_separator()).c_str());
|
||||
Tokenizer const tokens(env_var, separator);
|
||||
Tokenizer::const_iterator it = tokens.begin();
|
||||
Tokenizer::const_iterator const end = tokens.end();
|
||||
string const separator(1, os::path_separator());
|
||||
|
||||
vector<string> vars;
|
||||
for (; it != end; ++it)
|
||||
vars.push_back(os::internal_path(*it));
|
||||
|
||||
return vars;
|
||||
return getVectorFromString(env_var, separator);
|
||||
}
|
||||
|
||||
|
||||
@ -104,18 +94,13 @@ void setEnvPath(string const & name, vector<string> const & env)
|
||||
|
||||
void prependEnvPath(string const & name, string const & prefix)
|
||||
{
|
||||
string const separator(1, os::path_separator());
|
||||
vector<string> reversed_tokens
|
||||
= getVectorFromString(prefix, separator);
|
||||
vector<string> env_var = getEnvPath(name);
|
||||
|
||||
typedef boost::char_separator<char> Separator;
|
||||
typedef boost::tokenizer<Separator> Tokenizer;
|
||||
|
||||
Separator const separator(string(1, os::path_separator()).c_str());
|
||||
|
||||
// Prepend each new element to the list, removing identical elements
|
||||
// that occur later in the list.
|
||||
Tokenizer const tokens(prefix, separator);
|
||||
vector<string> reversed_tokens(tokens.begin(), tokens.end());
|
||||
|
||||
typedef vector<string>::const_reverse_iterator token_iterator;
|
||||
token_iterator it = reversed_tokens.rbegin();
|
||||
token_iterator const end = reversed_tokens.rend();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user