STXXL  1.4-dev
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
config.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  * lib/mng/config.cpp
3  *
4  * Part of the STXXL. See http://stxxl.sourceforge.net
5  *
6  * Copyright (C) 2002 Roman Dementiev <[email protected]>
7  * Copyright (C) 2007, 2009 Johannes Singler <[email protected]>
8  * Copyright (C) 2008, 2009 Andreas Beckmann <[email protected]>
9  * Copyright (C) 2013 Timo Bingmann <[email protected]>
10  *
11  * Distributed under the Boost Software License, Version 1.0.
12  * (See accompanying file LICENSE_1_0.txt or copy at
13  * http://www.boost.org/LICENSE_1_0.txt)
14  **************************************************************************/
15 
16 #include <fstream>
18 #include <stxxl/bits/config.h>
19 #include <stxxl/bits/io/file.h>
20 #include <stxxl/bits/mng/config.h>
22 #include <stxxl/version.h>
23 
24 #if STXXL_WINDOWS
25  #ifndef NOMINMAX
26  #define NOMINMAX
27  #endif
28  #include <windows.h>
29 #else
30  #include <unistd.h>
31 #endif
32 
34 
35 static inline bool exist_file(const std::string& path)
36 {
37  //STXXL_MSG("Checking " << path << " for disk configuration.");
38  std::ifstream in(path.c_str());
39  return in.good();
40 }
41 
42 config::~config()
43 {
44  for (disk_list_type::const_iterator it = disks_list.begin();
45  it != disks_list.end(); it++)
46  {
47  if (it->delete_on_exit)
48  {
49  STXXL_ERRMSG("Removing disk file: " << it->path);
50  unlink(it->path.c_str());
51  }
52  }
53 }
54 
55 void config::initialize()
56 {
57  // if disks_list is empty, then try to load disk configuration files
58  if (disks_list.size() == 0)
59  {
60  find_config();
61  }
62 
63  m_max_device_id = 0;
64 
65  is_initialized = true;
66 }
67 
68 void config::find_config()
69 {
70  // check several locations for disk configuration files
71 
72  // check STXXLCFG environment path
73  const char* stxxlcfg = getenv("STXXLCFG");
74  if (stxxlcfg && exist_file(stxxlcfg))
75  return load_config_file(stxxlcfg);
76 
77 #if !STXXL_WINDOWS
78  // read environment, unix style
79  const char* hostname = getenv("HOSTNAME");
80  const char* home = getenv("HOME");
81  const char* suffix = "";
82 #else
83  // read environment, windows style
84  const char* hostname = getenv("COMPUTERNAME");
85  const char* home = getenv("APPDATA");
86  const char* suffix = ".txt";
87 #endif
88 
89  // check current directory
90  {
91  std::string basepath = "./.stxxl";
92 
93  if (hostname && exist_file(basepath + "." + hostname + suffix))
94  return load_config_file(basepath + "." + hostname + suffix);
95 
96  if (exist_file(basepath + suffix))
97  return load_config_file(basepath + suffix);
98  }
99 
100  // check home directory
101  if (home)
102  {
103  std::string basepath = std::string(home) + "/.stxxl";
104 
105  if (hostname && exist_file(basepath + "." + hostname + suffix))
106  return load_config_file(basepath + "." + hostname + suffix);
107 
108  if (exist_file(basepath + suffix))
109  return load_config_file(basepath + suffix);
110  }
111 
112  // load default configuration
113  load_default_config();
114 }
115 
116 void config::load_default_config()
117 {
118  STXXL_ERRMSG("Warning: no config file found.");
119  STXXL_ERRMSG("Using default disk configuration.");
120 #if !STXXL_WINDOWS
121  disk_config entry1("/var/tmp/stxxl", 1000 * 1024 * 1024, "syscall");
122  entry1.delete_on_exit = true;
123  entry1.autogrow = true;
124 #else
125  disk_config entry1("", 1000 * 1024 * 1024, "wincall");
126  entry1.delete_on_exit = true;
127  entry1.autogrow = true;
128 
129  char* tmpstr = new char[255];
130  if (GetTempPathA(255, tmpstr) == 0)
131  STXXL_THROW_WIN_LASTERROR(resource_error, "GetTempPathA()");
132  entry1.path = tmpstr;
133  entry1.path += "stxxl.tmp";
134  delete[] tmpstr;
135 #endif
136  disks_list.push_back(entry1);
137 
138  // no flash disks
139  first_flash = (unsigned int)disks_list.size();
140 }
141 
142 void config::load_config_file(const std::string& config_path)
143 {
144  std::vector<disk_config> flash_list;
145  std::ifstream cfg_file(config_path.c_str());
146 
147  if (!cfg_file)
148  return load_default_config();
149 
150  std::string line;
151 
152  while (std::getline(cfg_file, line))
153  {
154  // skip comments
155  if (line.size() == 0 || line[0] == '#') continue;
156 
157  disk_config entry;
158  entry.parse_line(line); // throws on errors
159 
160  if (!entry.flash)
161  disks_list.push_back(entry);
162  else
163  flash_list.push_back(entry);
164  }
165  cfg_file.close();
166 
167  // put flash devices after regular disks
168  first_flash = (unsigned int)disks_list.size();
169  disks_list.insert(disks_list.end(), flash_list.begin(), flash_list.end());
170 
171  if (disks_list.empty()) {
172  STXXL_THROW(std::runtime_error,
173  "No disks found in '" << config_path << "'.");
174  }
175 }
176 
177 //! Returns automatic physical device id counter
178 unsigned int config::get_max_device_id()
179 {
180  return m_max_device_id;
181 }
182 
183 //! Returns next automatic physical device id counter
184 unsigned int config::get_next_device_id()
185 {
186  return m_max_device_id++;
187 }
188 
189 //! Update the automatic physical device id counter
190 void config::update_max_device_id(unsigned int devid)
191 {
192  if (m_max_device_id < devid + 1)
193  m_max_device_id = devid + 1;
194 }
195 
196 uint64 config::total_size() const
197 {
198  assert(is_initialized);
199 
200  uint64 total_size = 0;
201 
202  for (disk_list_type::const_iterator it = disks_list.begin();
203  it != disks_list.end(); it++)
204  {
205  total_size += it->size;
206  }
207 
208  return total_size;
209 }
210 
211 ////////////////////////////////////////////////////////////////////////////////
212 
213 disk_config::disk_config()
214  : size(0),
215  autogrow(true),
216  delete_on_exit(false),
217  direct(DIRECT_TRY),
218  flash(false),
219  queue(file::DEFAULT_QUEUE),
220  device_id(file::DEFAULT_DEVICE_ID),
221  raw_device(false),
222  unlink_on_open(false),
223  queue_length(0)
224 { }
225 
226 disk_config::disk_config(const std::string& _path, uint64 _size,
227  const std::string& _io_impl)
228  : path(_path),
229  size(_size),
230  io_impl(_io_impl),
231  autogrow(true),
232  delete_on_exit(false),
233  direct(DIRECT_TRY),
234  flash(false),
235  queue(file::DEFAULT_QUEUE),
236  device_id(file::DEFAULT_DEVICE_ID),
237  raw_device(false),
238  unlink_on_open(false),
239  queue_length(0)
240 {
241  parse_fileio();
242 }
243 
244 disk_config::disk_config(const std::string& line)
245  : size(0),
246  autogrow(true),
247  delete_on_exit(false),
248  direct(DIRECT_TRY),
249  flash(false),
250  queue(file::DEFAULT_QUEUE),
251  device_id(file::DEFAULT_DEVICE_ID),
252  raw_device(false),
253  unlink_on_open(false),
254  queue_length(0)
255 {
256  parse_line(line);
257 }
258 
259 void disk_config::parse_line(const std::string& line)
260 {
261  // split off disk= or flash=
262  std::vector<std::string> eqfield = split(line, "=", 2, 2);
263 
264  if (eqfield[0] == "disk") {
265  flash = false;
266  }
267  else if (eqfield[0] == "flash") {
268  flash = true;
269  }
270  else {
271  STXXL_THROW(std::runtime_error,
272  "Unknown configuration token " << eqfield[0]);
273  }
274 
275  // *** Set Default Extra Options ***
276 
277  autogrow = true; // was default for a long time, have to keep it this way
278  delete_on_exit = false;
279  direct = DIRECT_TRY;
280  // flash is already set
283  unlink_on_open = false;
284 
285  // *** Save Basic Options ***
286 
287  // split at commands, at least 3 fields
288  std::vector<std::string> cmfield = split(eqfield[1], ",", 3, 3);
289 
290  // path:
291  path = cmfield[0];
292  // replace ### -> pid in path
293  {
294  std::string::size_type pos;
295  if ((pos = path.find("###")) != std::string::npos)
296  {
297 #if !STXXL_WINDOWS
298  int pid = getpid();
299 #else
300  DWORD pid = GetCurrentProcessId();
301 #endif
302  path.replace(pos, 3, to_str(pid));
303  }
304  }
305 
306  // size: (default unit MiB)
307  if (!parse_SI_IEC_size(cmfield[1], size, 'M')) {
308  STXXL_THROW(std::runtime_error,
309  "Invalid disk size '" << cmfield[1] << "' in disk configuration file.");
310  }
311 
312  if (size == 0) {
313  autogrow = true;
314  delete_on_exit = true;
315  }
316 
317  // io_impl:
318  io_impl = cmfield[2];
319  parse_fileio();
320 }
321 
323 {
324  // skip over leading spaces
325  size_t leadspace = io_impl.find_first_not_of(' ');
326  if (leadspace > 0)
327  io_impl = io_impl.substr(leadspace);
328 
329  // split off extra fileio parameters
330  size_t spacepos = io_impl.find(' ');
331  if (spacepos == std::string::npos)
332  return; // no space in fileio
333 
334  // *** Parse Extra Fileio Parameters ***
335 
336  std::string paramstr = io_impl.substr(spacepos + 1);
337  io_impl = io_impl.substr(0, spacepos);
338 
339  std::vector<std::string> param = split(paramstr, " ");
340 
341  for (std::vector<std::string>::const_iterator p = param.begin();
342  p != param.end(); ++p)
343  {
344  // split at equal sign
345  std::vector<std::string> eq = split(*p, "=", 2, 2);
346 
347  // *** PLEASE try to keep the elseifs sorted by parameter name!
348  if (*p == "") {
349  // skip blank options
350  }
351  else if (*p == "autogrow" || *p == "noautogrow" || eq[0] == "autogrow")
352  {
353  // TODO: which fileio implementation support autogrow?
354 
355  if (*p == "autogrow") autogrow = true;
356  else if (*p == "noautogrow") autogrow = false;
357  else if (eq[1] == "off") autogrow = false;
358  else if (eq[1] == "on") autogrow = true;
359  else if (eq[1] == "no") autogrow = false;
360  else if (eq[1] == "yes") autogrow = true;
361  else
362  {
363  STXXL_THROW(std::runtime_error,
364  "Invalid parameter '" << *p << "' in disk configuration file.");
365  }
366  }
367  else if (*p == "delete" || *p == "delete_on_exit")
368  {
369  delete_on_exit = true;
370  }
371  else if (*p == "direct" || *p == "nodirect" || eq[0] == "direct")
372  {
373  // io_impl is not checked here, but I guess that is okay for DIRECT
374  // since it depends highly platform _and_ build-time configuration.
375 
376  if (*p == "direct") direct = DIRECT_ON; // force ON
377  else if (*p == "nodirect") direct = DIRECT_OFF; // force OFF
378  else if (eq[1] == "off") direct = DIRECT_OFF;
379  else if (eq[1] == "try") direct = DIRECT_TRY;
380  else if (eq[1] == "on") direct = DIRECT_ON;
381  else if (eq[1] == "no") direct = DIRECT_OFF;
382  else if (eq[1] == "yes") direct = DIRECT_ON;
383  else
384  {
385  STXXL_THROW(std::runtime_error,
386  "Invalid parameter '" << *p << "' in disk configuration file.");
387  }
388  }
389  else if (eq[0] == "queue")
390  {
391  if (io_impl == "linuxaio") {
392  STXXL_THROW(std::runtime_error, "Parameter '" << *p << "' invalid for fileio '" << io_impl << "' in disk configuration file.");
393  }
394 
395  char* endp;
396  queue = (int)strtoul(eq[1].c_str(), &endp, 10);
397  if (endp && *endp != 0) {
398  STXXL_THROW(std::runtime_error,
399  "Invalid parameter '" << *p << "' in disk configuration file.");
400  }
401  }
402  else if (eq[0] == "queue_length")
403  {
404  if (io_impl != "linuxaio") {
405  STXXL_THROW(std::runtime_error, "Parameter '" << *p << "' "
406  "is only valid for fileio linuxaio "
407  "in disk configuration file.");
408  }
409 
410  char* endp;
411  queue_length = (int)strtoul(eq[1].c_str(), &endp, 10);
412  if (endp && *endp != 0) {
413  STXXL_THROW(std::runtime_error,
414  "Invalid parameter '" << *p << "' in disk configuration file.");
415  }
416  }
417  else if (eq[0] == "device_id" || eq[0] == "devid")
418  {
419  char* endp;
420  device_id = (int)strtoul(eq[1].c_str(), &endp, 10);
421  if (endp && *endp != 0) {
422  STXXL_THROW(std::runtime_error,
423  "Invalid parameter '" << *p << "' in disk configuration file.");
424  }
425  }
426  else if (*p == "raw_device")
427  {
428  if (!(io_impl == "syscall")) {
429  STXXL_THROW(std::runtime_error, "Parameter '" << *p << "' invalid for fileio '" << io_impl << "' in disk configuration file.");
430  }
431 
432  raw_device = true;
433  }
434  else if (*p == "unlink" || *p == "unlink_on_open")
435  {
436  if (!(io_impl == "syscall" || io_impl == "linuxaio" ||
437  io_impl == "mmap" || io_impl == "wbtl"))
438  {
439  STXXL_THROW(std::runtime_error, "Parameter '" << *p << "' invalid for fileio '" << io_impl << "' in disk configuration file.");
440  }
441 
442  unlink_on_open = true;
443  }
444  else
445  {
446  STXXL_THROW(std::runtime_error,
447  "Invalid optional parameter '" << *p << "' in disk configuration file.");
448  }
449  }
450 }
451 
452 std::string disk_config::fileio_string() const
453 {
454  std::ostringstream oss;
455 
456  oss << io_impl;
457 
458  if (!autogrow)
459  oss << " autogrow=no";
460 
461  if (delete_on_exit)
462  oss << " delete_on_exit";
463 
464  // tristate direct variable: OFF, TRY, ON
465  if (direct == DIRECT_OFF)
466  oss << " direct=off";
467  else if (direct == DIRECT_TRY)
468  ; // silenced: oss << " direct=try";
469  else if (direct == DIRECT_ON)
470  oss << " direct=on";
471  else
472  STXXL_THROW(std::runtime_error, "Invalid setting for 'direct' option.");
473 
474  if (flash)
475  oss << " flash";
476 
478  oss << " queue=" << queue;
479 
481  oss << " devid=" << device_id;
482 
483  if (raw_device)
484  oss << " raw_device";
485 
486  if (unlink_on_open)
487  oss << " unlink_on_open";
488 
489  if (queue_length != 0)
490  oss << " queue_length=" << queue_length;
491 
492  return oss.str();
493 }
494 
496 // vim: et:ts=4:sw=4
static const unsigned int DEFAULT_DEVICE_ID
Definition: file.h:85
int queue
select request queue for disk. Use different queues for files on different disks. queue=-1 -&gt; default...
Definition: config.h:91
unsigned long long int uint64
Definition: types.h:39
External FIFO queue container. Introduction to queue container: see STXXL Queue tutorial Design a...
Definition: queue.h:51
bool unlink_on_open
unlink file immediately after opening (available on most Unix)
Definition: config.h:101
#define STXXL_THROW(exception_type, error_message)
Throws exception_type with &quot;Error in [function] : [error_message]&quot;.
static bool exist_file(const std::string &path)
Definition: config.cpp:35
#define STXXL_THROW_WIN_LASTERROR(exception_type, error_message)
Throws exception_type with &quot;Error in [function] : [error_message] : [formatted GetLastError()]&quot;.
bool delete_on_exit
delete file on program exit (default for autoconfigurated files)
Definition: config.h:79
Defines interface of file.
Definition: file.h:56
std::string path
the file path used by the io implementation
Definition: config.h:41
Encapsulate the configuration of one &quot;disk&quot;. The disk is actually a file I/O object which block_manag...
Definition: config.h:34
static std::vector< std::string > split(const std::string &str, const std::string &sep, unsigned int min_fields=0, unsigned int limit_fields=std::numeric_limits< unsigned int >::max())
Split a string by given separator string. Returns a vector of strings with at least min_fields and at...
Definition: utils.h:56
bool autogrow
autogrow file if more disk space is needed, automatically set if size == 0.
Definition: config.h:76
std::string to_str(const Type &t)
Format any ostream-able type into a string.
Definition: utils.h:97
#define STXXL_BEGIN_NAMESPACE
Definition: namespace.h:16
std::string fileio_string() const
return formatted fileio name and optional configuration parameters
Definition: config.cpp:452
void parse_line(const std::string &line)
parse a disk=&lt;path&gt;,&lt;size&gt;,&lt;fileio&gt; options line into disk_config, throws std::runtime_error on parse...
Definition: config.cpp:259
bool parse_SI_IEC_size(const std::string &str, uint64 &size, char def_unit=0)
Parse a string like &quot;343KB&quot; or &quot;44 GiB&quot; into the corresponding size in bytes. Returns the number of b...
Definition: utils.cpp:24
disk_config()
default constructor
Definition: config.cpp:213
unsigned int device_id
the selected physical device id (e.g. for calculating prefetching sequences). If -1 then the device i...
Definition: config.h:95
#define STXXL_ERRMSG(x)
Definition: verbose.h:94
int queue_length
desired queue length for linuxaio_file and linuxaio_queue
Definition: config.h:104
static const int DEFAULT_QUEUE
Definition: file.h:82
bool raw_device
turned on by syscall fileio when the path points to a raw block device
Definition: config.h:98
void parse_fileio()
parse the &quot;io_impl&quot; parameter into the optional parameter fields.
Definition: config.cpp:322
static const int DEFAULT_LINUXAIO_QUEUE
Definition: file.h:83
bool flash
marks flash drives (configuration entries with flash= instead of disk=)
Definition: config.h:87
enum stxxl::disk_config::direct_type direct
std::string io_impl
io implementation to access file
Definition: config.h:47
uint64 size
file size to initially allocate
Definition: config.h:44
#define STXXL_END_NAMESPACE
Definition: namespace.h:17