diff --git a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php
index 4ec61cf50..139684adb 100644
--- a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php
+++ b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php
@@ -1,221 +1,224 @@
<?php
final class PhabricatorExtraConfigSetupCheck extends PhabricatorSetupCheck {
public function getDefaultGroup() {
return self::GROUP_OTHER;
}
protected function executeChecks() {
$ancient_config = self::getAncientConfig();
$all_keys = PhabricatorEnv::getAllConfigKeys();
$all_keys = array_keys($all_keys);
sort($all_keys);
$defined_keys = PhabricatorApplicationConfigOptions::loadAllOptions();
foreach ($all_keys as $key) {
if (isset($defined_keys[$key])) {
continue;
}
if (isset($ancient_config[$key])) {
$summary = pht(
'This option has been removed. You may delete it at your '.
'convenience.');
$message = pht(
"The configuration option '%s' has been removed. You may delete ".
"it at your convenience.".
"\n\n%s",
$key,
$ancient_config[$key]);
$short = pht('Obsolete Config');
$name = pht('Obsolete Configuration Option "%s"', $key);
} else {
$summary = pht('This option is not recognized. It may be misspelled.');
$message = pht(
"The configuration option '%s' is not recognized. It may be ".
"misspelled, or it might have existed in an older version of ".
"Phabricator. It has no effect, and should be corrected or deleted.",
$key);
$short = pht('Unknown Config');
$name = pht('Unknown Configuration Option "%s"', $key);
}
$issue = $this->newIssue('config.unknown.'.$key)
->setShortName($short)
->setName($name)
->setSummary($summary);
$stack = PhabricatorEnv::getConfigSourceStack();
$stack = $stack->getStack();
$found = array();
$found_local = false;
$found_database = false;
foreach ($stack as $source_key => $source) {
$value = $source->getKeys(array($key));
if ($value) {
$found[] = $source->getName();
if ($source instanceof PhabricatorConfigDatabaseSource) {
$found_database = true;
}
if ($source instanceof PhabricatorConfigLocalSource) {
$found_local = true;
}
}
}
$message = $message."\n\n".pht(
'This configuration value is defined in these %d '.
'configuration source(s): %s.',
count($found),
implode(', ', $found));
$issue->setMessage($message);
if ($found_local) {
$command = csprintf('phabricator/ $ ./bin/config delete %s', $key);
$issue->addCommand($command);
}
if ($found_database) {
$issue->addPhabricatorConfig($key);
}
}
}
/**
* Return a map of deleted config options. Keys are option keys; values are
* explanations of what happened to the option.
*/
public static function getAncientConfig() {
$reason_auth = pht(
'This option has been migrated to the "Auth" application. Your old '.
'configuration is still in effect, but now stored in "Auth" instead of '.
'configuration. Going forward, you can manage authentication from '.
'the web UI.');
$auth_config = array(
'controller.oauth-registration',
'auth.password-auth-enabled',
'facebook.auth-enabled',
'facebook.registration-enabled',
'facebook.auth-permanent',
'facebook.application-id',
'facebook.application-secret',
'facebook.require-https-auth',
'github.auth-enabled',
'github.registration-enabled',
'github.auth-permanent',
'github.application-id',
'github.application-secret',
'google.auth-enabled',
'google.registration-enabled',
'google.auth-permanent',
'google.application-id',
'google.application-secret',
'ldap.auth-enabled',
'ldap.hostname',
'ldap.port',
'ldap.base_dn',
'ldap.search_attribute',
'ldap.search-first',
'ldap.username-attribute',
'ldap.real_name_attributes',
'ldap.activedirectory_domain',
'ldap.version',
'ldap.referrals',
'ldap.anonymous-user-name',
'ldap.anonymous-user-password',
'ldap.start-tls',
'disqus.auth-enabled',
'disqus.registration-enabled',
'disqus.auth-permanent',
'disqus.application-id',
'disqus.application-secret',
'phabricator.oauth-uri',
'phabricator.auth-enabled',
'phabricator.registration-enabled',
'phabricator.auth-permanent',
'phabricator.application-id',
'phabricator.application-secret',
);
$ancient_config = array_fill_keys($auth_config, $reason_auth);
$markup_reason = pht(
'Custom remarkup rules are now added by subclassing '.
'PhabricatorRemarkupCustomInlineRule or '.
'PhabricatorRemarkupCustomBlockRule.');
$session_reason = pht(
'Sessions now expire and are garbage collected rather than having an '.
'arbitrary concurrency limit.');
$differential_field_reason = pht(
'All Differential fields are now managed through the configuration '.
'option "%s". Use that option to configure which fields are shown.',
'differential.fields');
$ancient_config += array(
'phid.external-loaders' =>
pht(
'External loaders have been replaced. Extend `PhabricatorPHIDType` '.
'to implement new PHID and handle types.'),
'maniphest.custom-task-extensions-class' =>
pht(
'Maniphest fields are now loaded automatically. You can configure '.
'them with `maniphest.fields`.'),
'maniphest.custom-fields' =>
pht(
'Maniphest fields are now defined in '.
'`maniphest.custom-field-definitions`. Existing definitions have '.
'been migrated.'),
'differential.custom-remarkup-rules' => $markup_reason,
'differential.custom-remarkup-block-rules' => $markup_reason,
'auth.sshkeys.enabled' => pht(
'SSH keys are now actually useful, so they are always enabled.'),
'differential.anonymous-access' => pht(
'Phabricator now has meaningful global access controls. See '.
'`policy.allow-public`.'),
'celerity.resource-path' => pht(
'An alternate resource map is no longer supported. Instead, use '.
'multiple maps. See T4222.'),
'metamta.send-immediately' => pht(
'Mail is now always delivered by the daemons.'),
'auth.sessions.conduit' => $session_reason,
'auth.sessions.web' => $session_reason,
'tokenizer.ondemand' => pht(
'Phabricator now manages typeahead strategies automatically.'),
'differential.revision-custom-detail-renderer' => pht(
'Obsolete; use standard rendering events instead.'),
'differential.show-host-field' => $differential_field_reason,
'differential.show-test-plan-field' => $differential_field_reason,
'differential.field-selector' => $differential_field_reason,
'phabricator.show-beta-applications' => pht(
'This option has been renamed to `phabricator.show-prototypes` '.
'to emphasize the unfinished nature of many prototype applications. '.
'Your existing setting has been migrated.'),
'notification.user' => pht(
'The notification server no longer requires root permissions. Start '.
'the server as the user you want it to run under.'),
'notification.debug' => pht(
'Notifications no longer have a dedicated debugging mode.'),
'translation.provider' => pht(
'The translation implementation has changed and providers are no '.
'longer used or supported.'),
'config.mask' => pht(
'Use `config.hide` instead of this option.'),
'phd.start-taskmasters' => pht(
'Taskmasters now use an autoscaling pool. You can configure the '.
'pool size with `phd.taskmasters`.'),
'storage.engine-selector' => pht(
'Phabricator now automatically discovers available storage engines '.
'at runtime.'),
+ 'storage.upload-size-limit' => pht(
+ 'Phabricator now supports arbitrarily large files. Consult the '.
+ 'documentation for configuration details.'),
);
return $ancient_config;
}
}
diff --git a/src/applications/config/check/PhabricatorMySQLSetupCheck.php b/src/applications/config/check/PhabricatorMySQLSetupCheck.php
index 44d9cb3cd..6747158b8 100644
--- a/src/applications/config/check/PhabricatorMySQLSetupCheck.php
+++ b/src/applications/config/check/PhabricatorMySQLSetupCheck.php
@@ -1,340 +1,343 @@
<?php
final class PhabricatorMySQLSetupCheck extends PhabricatorSetupCheck {
public function getDefaultGroup() {
return self::GROUP_MYSQL;
}
public static function loadRawConfigValue($key) {
$conn_raw = id(new PhabricatorUser())->establishConnection('w');
try {
$value = queryfx_one($conn_raw, 'SELECT @@%Q', $key);
$value = $value['@@'.$key];
} catch (AphrontQueryException $ex) {
$value = null;
}
return $value;
}
protected function executeChecks() {
$max_allowed_packet = self::loadRawConfigValue('max_allowed_packet');
- $recommended_minimum = 1024 * 1024;
+
+ // This primarily supports setting the filesize limit for MySQL to 8MB,
+ // which may produce a >16MB packet after escaping.
+ $recommended_minimum = (32 * 1024 * 1024);
if ($max_allowed_packet < $recommended_minimum) {
$message = pht(
- "MySQL is configured with a very small 'max_allowed_packet' (%d), ".
+ "MySQL is configured with a small 'max_allowed_packet' (%d), ".
"which may cause some large writes to fail. Strongly consider raising ".
"this to at least %d in your MySQL configuration.",
$max_allowed_packet,
$recommended_minimum);
$this->newIssue('mysql.max_allowed_packet')
->setName(pht('Small MySQL "max_allowed_packet"'))
->setMessage($message)
->addMySQLConfig('max_allowed_packet');
}
$modes = self::loadRawConfigValue('sql_mode');
$modes = explode(',', $modes);
if (!in_array('STRICT_ALL_TABLES', $modes)) {
$summary = pht(
'MySQL is not in strict mode, but using strict mode is strongly '.
'encouraged.');
$message = pht(
"On your MySQL instance, the global %s is not set to %s. ".
"It is strongly encouraged that you enable this mode when running ".
"Phabricator.\n\n".
"By default MySQL will silently ignore some types of errors, which ".
"can cause data loss and raise security concerns. Enabling strict ".
"mode makes MySQL raise an explicit error instead, and prevents this ".
"entire class of problems from doing any damage.\n\n".
"You can find more information about this mode (and how to configure ".
"it) in the MySQL manual. Usually, it is sufficient to add this to ".
"your %s file (in the %s section) and then restart %s:\n\n".
"%s\n".
"(Note that if you run other applications against the same database, ".
"they may not work in strict mode. Be careful about enabling it in ".
"these cases.)",
phutil_tag('tt', array(), 'sql_mode'),
phutil_tag('tt', array(), 'STRICT_ALL_TABLES'),
phutil_tag('tt', array(), 'my.cnf'),
phutil_tag('tt', array(), '[mysqld]'),
phutil_tag('tt', array(), 'mysqld'),
phutil_tag('pre', array(), 'sql_mode=STRICT_ALL_TABLES'));
$this->newIssue('mysql.mode')
->setName(pht('MySQL STRICT_ALL_TABLES Mode Not Set'))
->setSummary($summary)
->setMessage($message)
->addMySQLConfig('sql_mode');
}
if (in_array('ONLY_FULL_GROUP_BY', $modes)) {
$summary = pht(
'MySQL is in ONLY_FULL_GROUP_BY mode, but using this mode is strongly '.
'discouraged.');
$message = pht(
"On your MySQL instance, the global %s is set to %s. ".
"It is strongly encouraged that you disable this mode when running ".
"Phabricator.\n\n".
"With %s enabled, MySQL rejects queries for which the select list ".
"or (as of MySQL 5.0.23) %s list refer to nonaggregated columns ".
"that are not named in the %s clause. More importantly, Phabricator ".
"does not work properly with this mode enabled.\n\n".
"You can find more information about this mode (and how to configure ".
"it) in the MySQL manual. Usually, it is sufficient to change the %s ".
"in your %s file (in the %s section) and then restart %s:\n\n".
"%s\n".
"(Note that if you run other applications against the same database, ".
"they may not work with %s. Be careful about enabling ".
"it in these cases and consider migrating Phabricator to a different ".
"database.)",
phutil_tag('tt', array(), 'sql_mode'),
phutil_tag('tt', array(), 'ONLY_FULL_GROUP_BY'),
phutil_tag('tt', array(), 'ONLY_FULL_GROUP_BY'),
phutil_tag('tt', array(), 'HAVING'),
phutil_tag('tt', array(), 'GROUP BY'),
phutil_tag('tt', array(), 'sql_mode'),
phutil_tag('tt', array(), 'my.cnf'),
phutil_tag('tt', array(), '[mysqld]'),
phutil_tag('tt', array(), 'mysqld'),
phutil_tag('pre', array(), 'sql_mode=STRICT_ALL_TABLES'),
phutil_tag('tt', array(), 'ONLY_FULL_GROUP_BY'));
$this->newIssue('mysql.mode')
->setName(pht('MySQL ONLY_FULL_GROUP_BY Mode Set'))
->setSummary($summary)
->setMessage($message)
->addMySQLConfig('sql_mode');
}
$stopword_file = self::loadRawConfigValue('ft_stopword_file');
if (!PhabricatorDefaultSearchEngineSelector::shouldUseElasticSearch()) {
if ($stopword_file === null) {
$summary = pht(
'Your version of MySQL does not support configuration of a '.
'stopword file. You will not be able to find search results for '.
'common words.');
$message = pht(
"Your MySQL instance does not support the %s option. You will not ".
"be able to find search results for common words. You can gain ".
"access to this option by upgrading MySQL to a more recent ".
"version.\n\n".
"You can ignore this warning if you plan to configure ElasticSearch ".
"later, or aren't concerned about searching for common words.",
phutil_tag('tt', array(), 'ft_stopword_file'));
$this->newIssue('mysql.ft_stopword_file')
->setName(pht('MySQL ft_stopword_file Not Supported'))
->setSummary($summary)
->setMessage($message)
->addMySQLConfig('ft_stopword_file');
} else if ($stopword_file == '(built-in)') {
$root = dirname(phutil_get_library_root('phabricator'));
$stopword_path = $root.'/resources/sql/stopwords.txt';
$stopword_path = Filesystem::resolvePath($stopword_path);
$namespace = PhabricatorEnv::getEnvConfig('storage.default-namespace');
$summary = pht(
'MySQL is using a default stopword file, which will prevent '.
'searching for many common words.');
$message = pht(
"Your MySQL instance is using the builtin stopword file for ".
"building search indexes. This can make Phabricator's search ".
"feature less useful.\n\n".
"Stopwords are common words which are not indexed and thus can not ".
"be searched for. The default stopword file has about 500 words, ".
"including various words which you are likely to wish to search ".
"for, such as 'various', 'likely', 'wish', and 'zero'.\n\n".
"To make search more useful, you can use an alternate stopword ".
"file with fewer words. Alternatively, if you aren't concerned ".
"about searching for common words, you can ignore this warning. ".
"If you later plan to configure ElasticSearch, you can also ignore ".
"this warning: this stopword file only affects MySQL fulltext ".
"indexes.\n\n".
"To choose a different stopword file, add this to your %s file ".
"(in the %s section) and then restart %s:\n\n".
"%s\n".
"(You can also use a different file if you prefer. The file ".
"suggested above has about 50 of the most common English words.)\n\n".
"Finally, run this command to rebuild indexes using the new ".
"rules:\n\n".
"%s",
phutil_tag('tt', array(), 'my.cnf'),
phutil_tag('tt', array(), '[mysqld]'),
phutil_tag('tt', array(), 'mysqld'),
phutil_tag('pre', array(), 'ft_stopword_file='.$stopword_path),
phutil_tag(
'pre',
array(),
"mysql> REPAIR TABLE {$namespace}_search.search_documentfield;"));
$this->newIssue('mysql.ft_stopword_file')
->setName(pht('MySQL is Using Default Stopword File'))
->setSummary($summary)
->setMessage($message)
->addMySQLConfig('ft_stopword_file');
}
}
$min_len = self::loadRawConfigValue('ft_min_word_len');
if ($min_len >= 4) {
if (!PhabricatorDefaultSearchEngineSelector::shouldUseElasticSearch()) {
$namespace = PhabricatorEnv::getEnvConfig('storage.default-namespace');
$summary = pht(
'MySQL is configured to only index words with at least %d '.
'characters.',
$min_len);
$message = pht(
"Your MySQL instance is configured to use the default minimum word ".
"length when building search indexes, which is 4. This means words ".
"which are only 3 characters long will not be indexed and can not ".
"be searched for.\n\n".
"For example, you will not be able to find search results for words ".
"like 'SMS', 'web', or 'DOS'.\n\n".
"You can change this setting to 3 to allow these words to be ".
"indexed. Alternatively, you can ignore this warning if you are ".
"not concerned about searching for 3-letter words. If you later ".
"plan to configure ElasticSearch, you can also ignore this warning: ".
"only MySQL fulltext search is affected.\n\n".
"To reduce the minimum word length to 3, add this to your %s file ".
"(in the %s section) and then restart %s:\n\n".
"%s\n".
"Finally, run this command to rebuild indexes using the new ".
"rules:\n\n".
"%s",
phutil_tag('tt', array(), 'my.cnf'),
phutil_tag('tt', array(), '[mysqld]'),
phutil_tag('tt', array(), 'mysqld'),
phutil_tag('pre', array(), 'ft_min_word_len=3'),
phutil_tag(
'pre',
array(),
"mysql> REPAIR TABLE {$namespace}_search.search_documentfield;"));
$this->newIssue('mysql.ft_min_word_len')
->setName(pht('MySQL is Using Default Minimum Word Length'))
->setSummary($summary)
->setMessage($message)
->addMySQLConfig('ft_min_word_len');
}
}
$bool_syntax = self::loadRawConfigValue('ft_boolean_syntax');
if ($bool_syntax != ' |-><()~*:""&^') {
if (!PhabricatorDefaultSearchEngineSelector::shouldUseElasticSearch()) {
$summary = pht(
'MySQL is configured to search on fulltext indexes using "OR" by '.
'default. Using "AND" is usually the desired behaviour.');
$message = pht(
"Your MySQL instance is configured to use the default Boolean ".
"search syntax when using fulltext indexes. This means searching ".
"for 'search words' will yield the query 'search OR words' ".
"instead of the desired 'search AND words'.\n\n".
"This might produce unexpected search results. \n\n".
"You can change this setting to a more sensible default. ".
"Alternatively, you can ignore this warning if ".
"using 'OR' is the desired behaviour. If you later plan ".
"to configure ElasticSearch, you can also ignore this warning: ".
"only MySQL fulltext search is affected.\n\n".
"To change this setting, add this to your %s file ".
"(in the %s section) and then restart %s:\n\n".
"%s\n",
phutil_tag('tt', array(), 'my.cnf'),
phutil_tag('tt', array(), '[mysqld]'),
phutil_tag('tt', array(), 'mysqld'),
phutil_tag('pre', array(), 'ft_boolean_syntax=\' |-><()~*:""&^\''));
$this->newIssue('mysql.ft_boolean_syntax')
->setName(pht('MySQL is Using the Default Boolean Syntax'))
->setSummary($summary)
->setMessage($message)
->addMySQLConfig('ft_boolean_syntax');
}
}
$innodb_pool = self::loadRawConfigValue('innodb_buffer_pool_size');
$innodb_bytes = phutil_parse_bytes($innodb_pool);
$innodb_readable = phutil_format_bytes($innodb_bytes);
// This is arbitrary and just trying to detect values that the user
// probably didn't set themselves. The Mac OS X default is 128MB and
// 40% of an AWS EC2 Micro instance is 245MB, so keeping it somewhere
// between those two values seems like a reasonable approximation.
$minimum_readable = '225MB';
$minimum_bytes = phutil_parse_bytes($minimum_readable);
if ($innodb_bytes < $minimum_bytes) {
$summary = pht(
'MySQL is configured with a very small innodb_buffer_pool_size, '.
'which may impact performance.');
$message = pht(
"Your MySQL instance is configured with a very small %s (%s). ".
"This may cause poor database performance and lock exhaustion.\n\n".
"There are no hard-and-fast rules to setting an appropriate value, ".
"but a reasonable starting point for a standard install is something ".
"like 40%% of the total memory on the machine. For example, if you ".
"have 4GB of RAM on the machine you have installed Phabricator on, ".
"you might set this value to %s.\n\n".
"You can read more about this option in the MySQL documentation to ".
"help you make a decision about how to configure it for your use ".
"case. There are no concerns specific to Phabricator which make it ".
"different from normal workloads with respect to this setting.\n\n".
"To adjust the setting, add something like this to your %s file (in ".
"the %s section), replacing %s with an appropriate value for your ".
"host and use case. Then restart %s:\n\n".
"%s\n".
"If you're satisfied with the current setting, you can safely ".
"ignore this setup warning.",
phutil_tag('tt', array(), 'innodb_buffer_pool_size'),
phutil_tag('tt', array(), $innodb_readable),
phutil_tag('tt', array(), '1600M'),
phutil_tag('tt', array(), 'my.cnf'),
phutil_tag('tt', array(), '[mysqld]'),
phutil_tag('tt', array(), '1600M'),
phutil_tag('tt', array(), 'mysqld'),
phutil_tag('pre', array(), 'innodb_buffer_pool_size=1600M'));
$this->newIssue('mysql.innodb_buffer_pool_size')
->setName(pht('MySQL May Run Slowly'))
->setSummary($summary)
->setMessage($message)
->addMySQLConfig('innodb_buffer_pool_size');
}
$ok = PhabricatorStorageManagementAPI::isCharacterSetAvailableOnConnection(
'utf8mb4',
id(new PhabricatorUser())->establishConnection('w'));
if (!$ok) {
$summary = pht(
'You are using an old version of MySQL, and should upgrade.');
$message = pht(
'You are using an old version of MySQL which has poor unicode '.
'support (it does not support the "utf8mb4" collation set). You will '.
'encounter limitations when working with some unicode data.'.
"\n\n".
'We strongly recommend you upgrade to MySQL 5.5 or newer.');
$this->newIssue('mysql.utf8mb4')
->setName(pht('Old MySQL Version'))
->setSummary($summary)
->setMessage($message);
}
}
}
diff --git a/src/applications/config/check/PhabricatorStorageSetupCheck.php b/src/applications/config/check/PhabricatorStorageSetupCheck.php
index 12296f03a..01844ce0f 100644
--- a/src/applications/config/check/PhabricatorStorageSetupCheck.php
+++ b/src/applications/config/check/PhabricatorStorageSetupCheck.php
@@ -1,107 +1,150 @@
<?php
final class PhabricatorStorageSetupCheck extends PhabricatorSetupCheck {
public function getDefaultGroup() {
return self::GROUP_OTHER;
}
/**
* @phutil-external-symbol class PhabricatorStartup
*/
protected function executeChecks() {
- $upload_limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
- if (!$upload_limit) {
+ $chunk_engine_active = false;
+
+ $engines = PhabricatorFileStorageEngine::loadWritableEngines();
+ foreach ($engines as $engine) {
+ if ($engine->isChunkEngine()) {
+ $chunk_engine_active = true;
+ break;
+ }
+ }
+
+ if (!$chunk_engine_active) {
+ $doc_href = PhabricatorEnv::getDocLink('Configuring File Storage');
+
$message = pht(
- 'The Phabricator file upload limit is not configured. You may only '.
- 'be able to upload very small files until you configure it, because '.
- 'some PHP default limits are very low (as low as 2MB).');
+ 'Large file storage has not been configured, which will limit '.
+ 'the maximum size of file uploads. See %s for '.
+ 'instructions on configuring uploads and storage.',
+ phutil_tag(
+ 'a',
+ array(
+ 'href' => $doc_href,
+ 'target' => '_blank',
+ ),
+ pht('Configuring File Storage')));
$this
- ->newIssue('config.storage.upload-size-limit')
- ->setShortName(pht('Upload Limit'))
- ->setName(pht('Upload Limit Not Yet Configured'))
- ->setMessage($message)
- ->addPhabricatorConfig('storage.upload-size-limit');
- } else {
- $memory_limit = PhabricatorStartup::getOldMemoryLimit();
- if ($memory_limit && ((int)$memory_limit > 0)) {
- $memory_limit_bytes = phutil_parse_bytes($memory_limit);
- $memory_usage_bytes = memory_get_usage();
- $upload_limit_bytes = phutil_parse_bytes($upload_limit);
-
- $available_bytes = ($memory_limit_bytes - $memory_usage_bytes);
-
- if ($upload_limit_bytes > $available_bytes) {
- $summary = pht(
- 'Your PHP memory limit is configured in a way that may prevent '.
- 'you from uploading large files.');
-
- $message = pht(
- 'When you upload a file via drag-and-drop or the API, the entire '.
- 'file is buffered into memory before being written to permanent '.
- 'storage. Phabricator needs memory available to store these '.
- 'files while they are uploaded, but PHP is currently configured '.
- 'to limit the available memory.'.
- "\n\n".
- 'Your Phabricator %s is currently set to a larger value (%s) than '.
- 'the amount of available memory (%s) that a PHP process has '.
- 'available to use, so uploads via drag-and-drop and the API will '.
- 'hit the memory limit before they hit other limits.'.
- "\n\n".
- '(Note that the application itself must also fit in available '.
- 'memory, so not all of the memory under the memory limit is '.
- 'available for buffering file uploads.)'.
- "\n\n".
- "The easiest way to resolve this issue is to set %s to %s in your ".
- "PHP configuration, to disable the memory limit. There is ".
- "usually little or no value to using this option to limit ".
- "Phabricator process memory.".
- "\n\n".
- "You can also increase the limit, or decrease %s, or ignore this ".
- "issue and accept that these upload mechanisms will be limited ".
- "in the size of files they can handle.",
- phutil_tag('tt', array(), 'storage.upload-size-limit'),
- phutil_format_bytes($upload_limit_bytes),
- phutil_format_bytes($available_bytes),
- phutil_tag('tt', array(), 'memory_limit'),
- phutil_tag('tt', array(), '-1'),
- phutil_tag('tt', array(), 'storage.upload-size-limit'));
-
- $this
- ->newIssue('php.memory_limit.upload')
- ->setName(pht('Memory Limit Restricts File Uploads'))
- ->setSummary($summary)
- ->setMessage($message)
- ->addPHPConfig('memory_limit')
- ->addPHPConfigOriginalValue('memory_limit', $memory_limit)
- ->addPhabricatorConfig('storage.upload-size-limit');
- }
+ ->newIssue('large-files')
+ ->setShortName(pht('Large Files'))
+ ->setName(pht('Large File Storage Not Configured'))
+ ->setMessage($message);
+ }
+
+ $post_max_size = ini_get('post_max_size');
+ if ($post_max_size && ((int)$post_max_size > 0)) {
+ $post_max_bytes = phutil_parse_bytes($post_max_size);
+ $post_max_need = (32 * 1024 * 1024) * 100;
+ if ($post_max_need > $post_max_bytes) {
+ $summary = pht(
+ 'Set %s in your PHP configuration to at least 32MB '.
+ 'to support large file uploads.',
+ phutil_tag('tt', array(), 'post_max_size'));
+
+ $message = pht(
+ 'Adjust %s in your PHP configuration to at least 32MB. When '.
+ 'set to smaller value, large file uploads may not work properly.',
+ phutil_tag('tt', array(), 'post_max_size'));
+
+ $this
+ ->newIssue('php.post_max_size')
+ ->setName(pht('PHP post_max_size Not Configured'))
+ ->setSummary($summary)
+ ->setMessage($message)
+ ->setGroup(self::GROUP_PHP)
+ ->addPHPConfig('post_max_size');
+ }
+ }
+
+ // This is somewhat arbitrary, but make sure we have enough headroom to
+ // upload a default file at the chunk threshold (8MB), which may be
+ // base64 encoded, then JSON encoded in the request, and may need to be
+ // held in memory in the raw and as a query string.
+ $need_bytes = (64 * 1024 * 1024);
+
+ $memory_limit = PhabricatorStartup::getOldMemoryLimit();
+ if ($memory_limit && ((int)$memory_limit > 0)) {
+ $memory_limit_bytes = phutil_parse_bytes($memory_limit);
+ $memory_usage_bytes = memory_get_usage();
+
+ $available_bytes = ($memory_limit_bytes - $memory_usage_bytes);
+
+ if ($need_bytes > $available_bytes) {
+ $summary = pht(
+ 'Your PHP memory limit is configured in a way that may prevent '.
+ 'you from uploading large files or handling large requests.');
+
+ $message = pht(
+ 'When you upload a file via drag-and-drop or the API, chunks must '.
+ 'be buffered into memory before being written to permanent '.
+ 'storage. Phabricator needs memory available to store these '.
+ 'chunks while they are uploaded, but PHP is currently configured '.
+ 'to severly limit the available memory.'.
+ "\n\n".
+ 'PHP processes currently have very little free memory available '.
+ '(%s). To work well, processes should have at least %s.'.
+ "\n\n".
+ '(Note that the application itself must also fit in available '.
+ 'memory, so not all of the memory under the memory limit is '.
+ 'available for running workloads.)'.
+ "\n\n".
+ "The easiest way to resolve this issue is to set %s to %s in your ".
+ "PHP configuration, to disable the memory limit. There is ".
+ "usually little or no value to using this option to limit ".
+ "Phabricator process memory.".
+ "\n\n".
+ "You can also increase the limit or ignore this issue and accept ".
+ "that you may encounter problems uploading large files and ".
+ "processing large requests.",
+ phutil_format_bytes($available_bytes),
+ phutil_format_bytes($need_bytes),
+ phutil_tag('tt', array(), 'memory_limit'),
+ phutil_tag('tt', array(), '-1'));
+
+ $this
+ ->newIssue('php.memory_limit.upload')
+ ->setName(pht('Memory Limit Restricts File Uploads'))
+ ->setSummary($summary)
+ ->setMessage($message)
+ ->setGroup(self::GROUP_PHP)
+ ->addPHPConfig('memory_limit')
+ ->addPHPConfigOriginalValue('memory_limit', $memory_limit);
}
}
$local_path = PhabricatorEnv::getEnvConfig('storage.local-disk.path');
if (!$local_path) {
return;
}
if (!Filesystem::pathExists($local_path) ||
!is_readable($local_path) ||
!is_writable($local_path)) {
$message = pht(
'Configured location for storing uploaded files on disk ("%s") does '.
'not exist, or is not readable or writable. Verify the directory '.
'exists and is readable and writable by the webserver.',
$local_path);
$this
->newIssue('config.storage.local-disk.path')
->setShortName(pht('Local Disk Storage'))
->setName(pht('Local Disk Storage Not Readable/Writable'))
->setMessage($message)
->addPhabricatorConfig('storage.local-disk.path');
}
}
}
diff --git a/src/applications/files/config/PhabricatorFilesConfigOptions.php b/src/applications/files/config/PhabricatorFilesConfigOptions.php
index 36add0f08..b448b0b5f 100644
--- a/src/applications/files/config/PhabricatorFilesConfigOptions.php
+++ b/src/applications/files/config/PhabricatorFilesConfigOptions.php
@@ -1,205 +1,185 @@
<?php
final class PhabricatorFilesConfigOptions
extends PhabricatorApplicationConfigOptions {
public function getName() {
return pht('Files');
}
public function getDescription() {
return pht('Configure files and file storage.');
}
public function getFontIcon() {
return 'fa-file';
}
public function getGroup() {
return 'apps';
}
public function getOptions() {
$viewable_default = array(
'image/jpeg' => 'image/jpeg',
'image/jpg' => 'image/jpg',
'image/png' => 'image/png',
'image/gif' => 'image/gif',
'text/plain' => 'text/plain; charset=utf-8',
'text/x-diff' => 'text/plain; charset=utf-8',
// ".ico" favicon files, which have mime type diversity. See:
// http://en.wikipedia.org/wiki/ICO_(file_format)#MIME_type
'image/x-ico' => 'image/x-icon',
'image/x-icon' => 'image/x-icon',
'image/vnd.microsoft.icon' => 'image/x-icon',
'audio/x-wav' => 'audio/x-wav',
'application/ogg' => 'application/ogg',
'audio/mpeg' => 'audio/mpeg',
);
$image_default = array(
'image/jpeg' => true,
'image/jpg' => true,
'image/png' => true,
'image/gif' => true,
'image/x-ico' => true,
'image/x-icon' => true,
'image/vnd.microsoft.icon' => true,
);
$audio_default = array(
'audio/x-wav' => true,
'application/ogg' => true,
'audio/mpeg' => true,
);
// largely lifted from http://en.wikipedia.org/wiki/Internet_media_type
$icon_default = array(
// audio file icon
'audio/basic' => 'fa-file-audio-o',
'audio/L24' => 'fa-file-audio-o',
'audio/mp4' => 'fa-file-audio-o',
'audio/mpeg' => 'fa-file-audio-o',
'audio/ogg' => 'fa-file-audio-o',
'audio/vorbis' => 'fa-file-audio-o',
'audio/vnd.rn-realaudio' => 'fa-file-audio-o',
'audio/vnd.wave' => 'fa-file-audio-o',
'audio/webm' => 'fa-file-audio-o',
// movie file icon
'video/mpeg' => 'fa-file-movie-o',
'video/mp4' => 'fa-file-movie-o',
'video/ogg' => 'fa-file-movie-o',
'video/quicktime' => 'fa-file-movie-o',
'video/webm' => 'fa-file-movie-o',
'video/x-matroska' => 'fa-file-movie-o',
'video/x-ms-wmv' => 'fa-file-movie-o',
'video/x-flv' => 'fa-file-movie-o',
// pdf file icon
'application/pdf' => 'fa-file-pdf-o',
// zip file icon
'application/zip' => 'fa-file-zip-o',
// msword icon
'application/msword' => 'fa-file-word-o',
// msexcel
'application/vnd.ms-excel' => 'fa-file-excel-o',
// mspowerpoint
'application/vnd.ms-powerpoint' => 'fa-file-powerpoint-o',
) + array_fill_keys(array_keys($image_default), 'fa-file-image-o');
return array(
$this->newOption('files.viewable-mime-types', 'wild', $viewable_default)
->setSummary(
pht('Configure which MIME types are viewable in the browser.'))
->setDescription(
pht(
'Configure which uploaded file types may be viewed directly '.
'in the browser. Other file types will be downloaded instead '.
'of displayed. This is mainly a usability consideration, since '.
'browsers tend to freak out when viewing enormous binary files.'.
"\n\n".
'The keys in this map are vieweable MIME types; the values are '.
'the MIME types they are delivered as when they are viewed in '.
'the browser.')),
$this->newOption('files.image-mime-types', 'set', $image_default)
->setSummary(pht('Configure which MIME types are images.'))
->setDescription(
pht(
'List of MIME types which can be used as the `src` for an '.
'`<img />` tag.')),
$this->newOption('files.audio-mime-types', 'set', $audio_default)
->setSummary(pht('Configure which MIME types are audio.'))
->setDescription(
pht(
'List of MIME types which can be used to render an '.
'`<audio />` tag.')),
$this->newOption('files.icon-mime-types', 'wild', $icon_default)
->setSummary(pht('Configure which MIME types map to which icons.'))
->setDescription(
pht(
'Map of MIME type to icon name. MIME types which can not be '.
'found default to icon `doc_files`.')),
$this->newOption('storage.mysql-engine.max-size', 'int', 1000000)
->setSummary(
pht(
'Configure the largest file which will be put into the MySQL '.
'storage engine.')),
$this->newOption('storage.local-disk.path', 'string', null)
->setLocked(true)
->setSummary(pht('Local storage disk path.'))
->setDescription(
pht(
"Phabricator provides a local disk storage engine, which just ".
"writes files to some directory on local disk. The webserver ".
"must have read/write permissions on this directory. This is ".
"straightforward and suitable for most installs, but will not ".
"scale past one web frontend unless the path is actually an NFS ".
"mount, since you'll end up with some of the files written to ".
"each web frontend and no way for them to share. To use the ".
"local disk storage engine, specify the path to a directory ".
"here. To disable it, specify null.")),
$this->newOption('storage.s3.bucket', 'string', null)
->setSummary(pht('Amazon S3 bucket.'))
->setDescription(
pht(
"Set this to a valid Amazon S3 bucket to store files there. You ".
"must also configure S3 access keys in the 'Amazon Web Services' ".
"group.")),
- $this->newOption('storage.upload-size-limit', 'string', null)
- ->setSummary(
- pht('Limit to users in interfaces which allow uploading.'))
- ->setDescription(
- pht(
- "Set the size of the largest file a user may upload. This is ".
- "used to render text like 'Maximum file size: 10MB' on ".
- "interfaces where users can upload files, and files larger than ".
- "this size will be rejected. \n\n".
- "NOTE: **Setting this to a large size is NOT sufficient to ".
- "allow users to upload large files. You must also configure a ".
- "number of other settings.** To configure file upload limits, ".
- "consult the article 'Configuring File Upload Limits' in the ".
- "documentation. Once you've configured some limit across all ".
- "levels of the server, you can set this limit to an appropriate ".
- "value and the UI will then reflect the actual configured ".
- "limit.\n\n".
- "Specify this limit in bytes, or using a 'K', 'M', or 'G' ".
- "suffix."))
- ->addExample('10M', pht('Allow Uploads 10MB or Smaller')),
$this->newOption(
'metamta.files.public-create-email',
'string',
null)
->setLocked(true)
->setLockedMessage(pht(
'This configuration is deprecated. See description for details.'))
->setSummary(pht('DEPRECATED - Allow uploaded files via email.'))
->setDescription(
pht(
'This config has been deprecated in favor of [[ '.
'/applications/view/PhabricatorFilesApplication/ | '.
'application settings ]], which allow for multiple email '.
'addresses and other functionality.')),
$this->newOption(
'metamta.files.subject-prefix',
'string',
'[File]')
->setDescription(pht('Subject prefix for Files email.')),
$this->newOption('files.enable-imagemagick', 'bool', false)
->setBoolOptions(
array(
pht('Enable'),
pht('Disable'),
))
->setDescription(
pht(
'This option will use Imagemagick to rescale images, so animated '.
'GIFs can be thumbnailed and set as profile pictures. Imagemagick '.
'must be installed and the "convert" binary must be available to '.
'the webserver for this to work.')),
);
}
}
diff --git a/src/applications/files/controller/PhabricatorFileUploadController.php b/src/applications/files/controller/PhabricatorFileUploadController.php
index 063117075..1528b9d50 100644
--- a/src/applications/files/controller/PhabricatorFileUploadController.php
+++ b/src/applications/files/controller/PhabricatorFileUploadController.php
@@ -1,126 +1,104 @@
<?php
final class PhabricatorFileUploadController extends PhabricatorFileController {
public function processRequest() {
$request = $this->getRequest();
$viewer = $request->getUser();
$file = PhabricatorFile::initializeNewFile();
$e_file = true;
$errors = array();
if ($request->isFormPost()) {
$view_policy = $request->getStr('viewPolicy');
if (!$request->getFileExists('file')) {
$e_file = pht('Required');
$errors[] = pht('You must select a file to upload.');
} else {
$file = PhabricatorFile::newFromPHPUpload(
idx($_FILES, 'file'),
array(
'name' => $request->getStr('name'),
'authorPHID' => $viewer->getPHID(),
'viewPolicy' => $view_policy,
'isExplicitUpload' => true,
));
}
if (!$errors) {
return id(new AphrontRedirectResponse())->setURI($file->getInfoURI());
}
$file->setViewPolicy($view_policy);
}
$support_id = celerity_generate_unique_node_id();
$instructions = id(new AphrontFormMarkupControl())
->setControlID($support_id)
->setControlStyle('display: none')
->setValue(hsprintf(
'<br /><br /><strong>%s</strong> %s<br /><br />',
pht('Drag and Drop:'),
pht(
'You can also upload files by dragging and dropping them from your '.
'desktop onto this page or the Phabricator home page.')));
$policies = id(new PhabricatorPolicyQuery())
->setViewer($viewer)
->setObject($file)
->execute();
$form = id(new AphrontFormView())
->setUser($viewer)
->setEncType('multipart/form-data')
->appendChild(
id(new AphrontFormFileControl())
->setLabel(pht('File'))
->setName('file')
- ->setError($e_file)
- ->setCaption($this->renderUploadLimit()))
+ ->setError($e_file))
->appendChild(
id(new AphrontFormTextControl())
->setLabel(pht('Name'))
->setName('name')
->setValue($request->getStr('name')))
->appendChild(
id(new AphrontFormPolicyControl())
->setUser($viewer)
->setCapability(PhabricatorPolicyCapability::CAN_VIEW)
->setPolicyObject($file)
->setPolicies($policies)
->setName('viewPolicy'))
->appendChild(
id(new AphrontFormSubmitControl())
->setValue(pht('Upload'))
->addCancelButton('/file/'))
->appendChild($instructions);
$crumbs = $this->buildApplicationCrumbs();
$crumbs->addTextCrumb(pht('Upload'), $request->getRequestURI());
$title = pht('Upload File');
$global_upload = id(new PhabricatorGlobalUploadTargetView())
->setUser($viewer)
->setShowIfSupportedID($support_id);
$form_box = id(new PHUIObjectBoxView())
->setHeaderText($title)
->setFormErrors($errors)
->setForm($form);
return $this->buildApplicationPage(
array(
$crumbs,
$form_box,
$global_upload,
),
array(
'title' => $title,
));
}
- private function renderUploadLimit() {
- $limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
- $limit = phutil_parse_bytes($limit);
- if ($limit) {
- $formatted = phutil_format_bytes($limit);
- return 'Maximum file size: '.$formatted;
- }
-
- $doc_href = PhabricatorEnv::getDocLink(
- 'Configuring File Upload Limits');
- $doc_link = phutil_tag(
- 'a',
- array(
- 'href' => $doc_href,
- 'target' => '_blank',
- ),
- 'Configuring File Upload Limits');
-
- return hsprintf('Upload limit is not configured, see %s.', $doc_link);
- }
-
}
diff --git a/src/applications/files/exception/PhabricatorFileUploadException.php b/src/applications/files/exception/PhabricatorFileUploadException.php
index a856ffb2d..59d24ed29 100644
--- a/src/applications/files/exception/PhabricatorFileUploadException.php
+++ b/src/applications/files/exception/PhabricatorFileUploadException.php
@@ -1,33 +1,28 @@
<?php
final class PhabricatorFileUploadException extends Exception {
public function __construct($code) {
$map = array(
UPLOAD_ERR_INI_SIZE =>
pht("Uploaded file is too large: current limit is %s. To adjust ".
"this limit change 'upload_max_filesize' in php.ini.",
ini_get('upload_max_filesize')),
UPLOAD_ERR_FORM_SIZE =>
'File is too large.',
UPLOAD_ERR_PARTIAL =>
'File was only partially transferred, upload did not complete.',
UPLOAD_ERR_NO_FILE =>
'No file was uploaded.',
UPLOAD_ERR_NO_TMP_DIR =>
'Unable to write file: temporary directory does not exist.',
UPLOAD_ERR_CANT_WRITE =>
'Unable to write file: failed to write to temporary directory.',
UPLOAD_ERR_EXTENSION =>
'Unable to upload: a PHP extension stopped the upload.',
-
- -1000 =>
- pht("Uploaded file is too large: current limit is %s. To adjust this ".
- "limit change 'storage.upload-size-limit' in the Phabricator config.",
- PhabricatorEnv::getEnvConfig('storage.upload-size-limit')),
);
$message = idx($map, $code, 'Upload failed: unknown error.');
parent::__construct($message, $code);
}
}
diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php
index aa681f323..f790b2b5f 100644
--- a/src/applications/files/storage/PhabricatorFile.php
+++ b/src/applications/files/storage/PhabricatorFile.php
@@ -1,1327 +1,1312 @@
<?php
/**
* Parameters
* ==========
*
* When creating a new file using a method like @{method:newFromFileData}, these
* parameters are supported:
*
* | name | Human readable filename.
* | authorPHID | User PHID of uploader.
* | ttl | Temporary file lifetime, in seconds.
* | viewPolicy | File visibility policy.
* | isExplicitUpload | Used to show users files they explicitly uploaded.
* | canCDN | Allows the file to be cached and delivered over a CDN.
* | mime-type | Optional, explicit file MIME type.
* | builtin | Optional filename, identifies this as a builtin.
*
*/
final class PhabricatorFile extends PhabricatorFileDAO
implements
PhabricatorApplicationTransactionInterface,
PhabricatorTokenReceiverInterface,
PhabricatorSubscribableInterface,
PhabricatorFlaggableInterface,
PhabricatorPolicyInterface,
PhabricatorDestructibleInterface {
const ONETIME_TEMPORARY_TOKEN_TYPE = 'file:onetime';
const STORAGE_FORMAT_RAW = 'raw';
const METADATA_IMAGE_WIDTH = 'width';
const METADATA_IMAGE_HEIGHT = 'height';
const METADATA_CAN_CDN = 'canCDN';
const METADATA_BUILTIN = 'builtin';
const METADATA_PARTIAL = 'partial';
protected $name;
protected $mimeType;
protected $byteSize;
protected $authorPHID;
protected $secretKey;
protected $contentHash;
protected $metadata = array();
protected $mailKey;
protected $storageEngine;
protected $storageFormat;
protected $storageHandle;
protected $ttl;
protected $isExplicitUpload = 1;
protected $viewPolicy = PhabricatorPolicies::POLICY_USER;
protected $isPartial = 0;
private $objects = self::ATTACHABLE;
private $objectPHIDs = self::ATTACHABLE;
private $originalFile = self::ATTACHABLE;
public static function initializeNewFile() {
$app = id(new PhabricatorApplicationQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withClasses(array('PhabricatorFilesApplication'))
->executeOne();
$view_policy = $app->getPolicy(
FilesDefaultViewCapability::CAPABILITY);
return id(new PhabricatorFile())
->setViewPolicy($view_policy)
->setIsPartial(0)
->attachOriginalFile(null)
->attachObjects(array())
->attachObjectPHIDs(array());
}
protected function getConfiguration() {
return array(
self::CONFIG_AUX_PHID => true,
self::CONFIG_SERIALIZATION => array(
'metadata' => self::SERIALIZATION_JSON,
),
self::CONFIG_COLUMN_SCHEMA => array(
'name' => 'text255?',
'mimeType' => 'text255?',
'byteSize' => 'uint64',
'storageEngine' => 'text32',
'storageFormat' => 'text32',
'storageHandle' => 'text255',
'authorPHID' => 'phid?',
'secretKey' => 'bytes20?',
'contentHash' => 'bytes40?',
'ttl' => 'epoch?',
'isExplicitUpload' => 'bool?',
'mailKey' => 'bytes20',
'isPartial' => 'bool',
),
self::CONFIG_KEY_SCHEMA => array(
'key_phid' => null,
'phid' => array(
'columns' => array('phid'),
'unique' => true,
),
'authorPHID' => array(
'columns' => array('authorPHID'),
),
'contentHash' => array(
'columns' => array('contentHash'),
),
'key_ttl' => array(
'columns' => array('ttl'),
),
'key_dateCreated' => array(
'columns' => array('dateCreated'),
),
'key_partial' => array(
'columns' => array('authorPHID', 'isPartial'),
),
),
) + parent::getConfiguration();
}
public function generatePHID() {
return PhabricatorPHID::generateNewPHID(
PhabricatorFileFilePHIDType::TYPECONST);
}
public function save() {
if (!$this->getSecretKey()) {
$this->setSecretKey($this->generateSecretKey());
}
if (!$this->getMailKey()) {
$this->setMailKey(Filesystem::readRandomCharacters(20));
}
return parent::save();
}
public function getMonogram() {
return 'F'.$this->getID();
}
public static function readUploadedFileData($spec) {
if (!$spec) {
throw new Exception('No file was uploaded!');
}
$err = idx($spec, 'error');
if ($err) {
throw new PhabricatorFileUploadException($err);
}
$tmp_name = idx($spec, 'tmp_name');
$is_valid = @is_uploaded_file($tmp_name);
if (!$is_valid) {
throw new Exception('File is not an uploaded file.');
}
$file_data = Filesystem::readFile($tmp_name);
$file_size = idx($spec, 'size');
if (strlen($file_data) != $file_size) {
throw new Exception('File size disagrees with uploaded size.');
}
- self::validateFileSize(strlen($file_data));
-
return $file_data;
}
public static function newFromPHPUpload($spec, array $params = array()) {
$file_data = self::readUploadedFileData($spec);
$file_name = nonempty(
idx($params, 'name'),
idx($spec, 'name'));
$params = array(
'name' => $file_name,
) + $params;
return self::newFromFileData($file_data, $params);
}
public static function newFromXHRUpload($data, array $params = array()) {
- self::validateFileSize(strlen($data));
return self::newFromFileData($data, $params);
}
- private static function validateFileSize($size) {
- $limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
- if (!$limit) {
- return;
- }
-
- $limit = phutil_parse_bytes($limit);
- if ($size > $limit) {
- throw new PhabricatorFileUploadException(-1000);
- }
- }
-
/**
* Given a block of data, try to load an existing file with the same content
* if one exists. If it does not, build a new file.
*
* This method is generally used when we have some piece of semi-trusted data
* like a diff or a file from a repository that we want to show to the user.
* We can't just dump it out because it may be dangerous for any number of
* reasons; instead, we need to serve it through the File abstraction so it
* ends up on the CDN domain if one is configured and so on. However, if we
* simply wrote a new file every time we'd potentially end up with a lot
* of redundant data in file storage.
*
* To solve these problems, we use file storage as a cache and reuse the
* same file again if we've previously written it.
*
* NOTE: This method unguards writes.
*
* @param string Raw file data.
* @param dict Dictionary of file information.
*/
public static function buildFromFileDataOrHash(
$data,
array $params = array()) {
$file = id(new PhabricatorFile())->loadOneWhere(
'name = %s AND contentHash = %s LIMIT 1',
self::normalizeFileName(idx($params, 'name')),
self::hashFileContent($data));
if (!$file) {
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
$file = PhabricatorFile::newFromFileData($data, $params);
unset($unguarded);
}
return $file;
}
public static function newFileFromContentHash($hash, array $params) {
// Check to see if a file with same contentHash exist
$file = id(new PhabricatorFile())->loadOneWhere(
'contentHash = %s LIMIT 1',
$hash);
if ($file) {
// copy storageEngine, storageHandle, storageFormat
$copy_of_storage_engine = $file->getStorageEngine();
$copy_of_storage_handle = $file->getStorageHandle();
$copy_of_storage_format = $file->getStorageFormat();
$copy_of_byte_size = $file->getByteSize();
$copy_of_mime_type = $file->getMimeType();
$new_file = PhabricatorFile::initializeNewFile();
$new_file->setByteSize($copy_of_byte_size);
$new_file->setContentHash($hash);
$new_file->setStorageEngine($copy_of_storage_engine);
$new_file->setStorageHandle($copy_of_storage_handle);
$new_file->setStorageFormat($copy_of_storage_format);
$new_file->setMimeType($copy_of_mime_type);
$new_file->copyDimensions($file);
$new_file->readPropertiesFromParameters($params);
$new_file->save();
return $new_file;
}
return $file;
}
public static function newChunkedFile(
PhabricatorFileStorageEngine $engine,
$length,
array $params) {
$file = PhabricatorFile::initializeNewFile();
$file->setByteSize($length);
// TODO: We might be able to test the first chunk in order to figure
// this out more reliably, since MIME detection usually examines headers.
// However, enormous files are probably always either actually raw data
// or reasonable to treat like raw data.
$file->setMimeType('application/octet-stream');
$chunked_hash = idx($params, 'chunkedHash');
if ($chunked_hash) {
$file->setContentHash($chunked_hash);
} else {
// See PhabricatorChunkedFileStorageEngine::getChunkedHash() for some
// discussion of this.
$seed = Filesystem::readRandomBytes(64);
$hash = PhabricatorChunkedFileStorageEngine::getChunkedHashForInput(
$seed);
$file->setContentHash($hash);
}
$file->setStorageEngine($engine->getEngineIdentifier());
$file->setStorageHandle(PhabricatorFileChunk::newChunkHandle());
$file->setStorageFormat(self::STORAGE_FORMAT_RAW);
$file->setIsPartial(1);
$file->readPropertiesFromParameters($params);
return $file;
}
private static function buildFromFileData($data, array $params = array()) {
if (isset($params['storageEngines'])) {
$engines = $params['storageEngines'];
} else {
$size = strlen($data);
$engines = PhabricatorFileStorageEngine::loadStorageEngines($size);
if (!$engines) {
throw new Exception(
pht(
'No configured storage engine can store this file. See '.
'"Configuring File Storage" in the documentation for '.
'information on configuring storage engines.'));
}
}
assert_instances_of($engines, 'PhabricatorFileStorageEngine');
if (!$engines) {
throw new Exception(pht('No valid storage engines are available!'));
}
$file = PhabricatorFile::initializeNewFile();
$data_handle = null;
$engine_identifier = null;
$exceptions = array();
foreach ($engines as $engine) {
$engine_class = get_class($engine);
try {
list($engine_identifier, $data_handle) = $file->writeToEngine(
$engine,
$data,
$params);
// We stored the file somewhere so stop trying to write it to other
// places.
break;
} catch (PhabricatorFileStorageConfigurationException $ex) {
// If an engine is outright misconfigured (or misimplemented), raise
// that immediately since it probably needs attention.
throw $ex;
} catch (Exception $ex) {
phlog($ex);
// If an engine doesn't work, keep trying all the other valid engines
// in case something else works.
$exceptions[$engine_class] = $ex;
}
}
if (!$data_handle) {
throw new PhutilAggregateException(
'All storage engines failed to write file:',
$exceptions);
}
$file->setByteSize(strlen($data));
$file->setContentHash(self::hashFileContent($data));
$file->setStorageEngine($engine_identifier);
$file->setStorageHandle($data_handle);
// TODO: This is probably YAGNI, but allows for us to do encryption or
// compression later if we want.
$file->setStorageFormat(self::STORAGE_FORMAT_RAW);
$file->readPropertiesFromParameters($params);
if (!$file->getMimeType()) {
$tmp = new TempFile();
Filesystem::writeFile($tmp, $data);
$file->setMimeType(Filesystem::getMimeType($tmp));
}
try {
$file->updateDimensions(false);
} catch (Exception $ex) {
// Do nothing
}
$file->save();
return $file;
}
public static function newFromFileData($data, array $params = array()) {
$hash = self::hashFileContent($data);
$file = self::newFileFromContentHash($hash, $params);
if ($file) {
return $file;
}
return self::buildFromFileData($data, $params);
}
public function migrateToEngine(PhabricatorFileStorageEngine $engine) {
if (!$this->getID() || !$this->getStorageHandle()) {
throw new Exception(
"You can not migrate a file which hasn't yet been saved.");
}
$data = $this->loadFileData();
$params = array(
'name' => $this->getName(),
);
list($new_identifier, $new_handle) = $this->writeToEngine(
$engine,
$data,
$params);
$old_engine = $this->instantiateStorageEngine();
$old_identifier = $this->getStorageEngine();
$old_handle = $this->getStorageHandle();
$this->setStorageEngine($new_identifier);
$this->setStorageHandle($new_handle);
$this->save();
$this->deleteFileDataIfUnused(
$old_engine,
$old_identifier,
$old_handle);
return $this;
}
private function writeToEngine(
PhabricatorFileStorageEngine $engine,
$data,
array $params) {
$engine_class = get_class($engine);
$data_handle = $engine->writeFile($data, $params);
if (!$data_handle || strlen($data_handle) > 255) {
// This indicates an improperly implemented storage engine.
throw new PhabricatorFileStorageConfigurationException(
"Storage engine '{$engine_class}' executed writeFile() but did ".
"not return a valid handle ('{$data_handle}') to the data: it ".
"must be nonempty and no longer than 255 characters.");
}
$engine_identifier = $engine->getEngineIdentifier();
if (!$engine_identifier || strlen($engine_identifier) > 32) {
throw new PhabricatorFileStorageConfigurationException(
"Storage engine '{$engine_class}' returned an improper engine ".
"identifier '{$engine_identifier}': it must be nonempty ".
"and no longer than 32 characters.");
}
return array($engine_identifier, $data_handle);
}
public static function newFromFileDownload($uri, array $params = array()) {
// Make sure we're allowed to make a request first
if (!PhabricatorEnv::getEnvConfig('security.allow-outbound-http')) {
throw new Exception('Outbound HTTP requests are disabled!');
}
$uri = new PhutilURI($uri);
$protocol = $uri->getProtocol();
switch ($protocol) {
case 'http':
case 'https':
break;
default:
// Make sure we are not accessing any file:// URIs or similar.
return null;
}
$timeout = 5;
list($file_data) = id(new HTTPSFuture($uri))
->setTimeout($timeout)
->resolvex();
$params = $params + array(
'name' => basename($uri),
);
return self::newFromFileData($file_data, $params);
}
public static function normalizeFileName($file_name) {
$pattern = "@[\\x00-\\x19#%&+!~'\$\"\/=\\\\?<> ]+@";
$file_name = preg_replace($pattern, '_', $file_name);
$file_name = preg_replace('@_+@', '_', $file_name);
$file_name = trim($file_name, '_');
$disallowed_filenames = array(
'.' => 'dot',
'..' => 'dotdot',
'' => 'file',
);
$file_name = idx($disallowed_filenames, $file_name, $file_name);
return $file_name;
}
public function delete() {
// We want to delete all the rows which mark this file as the transformation
// of some other file (since we're getting rid of it). We also delete all
// the transformations of this file, so that a user who deletes an image
// doesn't need to separately hunt down and delete a bunch of thumbnails and
// resizes of it.
$outbound_xforms = id(new PhabricatorFileQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withTransforms(
array(
array(
'originalPHID' => $this->getPHID(),
'transform' => true,
),
))
->execute();
foreach ($outbound_xforms as $outbound_xform) {
$outbound_xform->delete();
}
$inbound_xforms = id(new PhabricatorTransformedFile())->loadAllWhere(
'transformedPHID = %s',
$this->getPHID());
$this->openTransaction();
foreach ($inbound_xforms as $inbound_xform) {
$inbound_xform->delete();
}
$ret = parent::delete();
$this->saveTransaction();
$this->deleteFileDataIfUnused(
$this->instantiateStorageEngine(),
$this->getStorageEngine(),
$this->getStorageHandle());
return $ret;
}
/**
* Destroy stored file data if there are no remaining files which reference
* it.
*/
public function deleteFileDataIfUnused(
PhabricatorFileStorageEngine $engine,
$engine_identifier,
$handle) {
// Check to see if any files are using storage.
$usage = id(new PhabricatorFile())->loadAllWhere(
'storageEngine = %s AND storageHandle = %s LIMIT 1',
$engine_identifier,
$handle);
// If there are no files using the storage, destroy the actual storage.
if (!$usage) {
try {
$engine->deleteFile($handle);
} catch (Exception $ex) {
// In the worst case, we're leaving some data stranded in a storage
// engine, which is not a big deal.
phlog($ex);
}
}
}
public static function hashFileContent($data) {
return sha1($data);
}
public function loadFileData() {
$engine = $this->instantiateStorageEngine();
$data = $engine->readFile($this->getStorageHandle());
switch ($this->getStorageFormat()) {
case self::STORAGE_FORMAT_RAW:
$data = $data;
break;
default:
throw new Exception('Unknown storage format.');
}
return $data;
}
/**
* Return an iterable which emits file content bytes.
*
* @param int Offset for the start of data.
* @param int Offset for the end of data.
* @return Iterable Iterable object which emits requested data.
*/
public function getFileDataIterator($begin = null, $end = null) {
$engine = $this->instantiateStorageEngine();
return $engine->getFileDataIterator($this, $begin, $end);
}
public function getViewURI() {
if (!$this->getPHID()) {
throw new Exception(
'You must save a file before you can generate a view URI.');
}
return $this->getCDNURI(null);
}
private function getCDNURI($token) {
$name = phutil_escape_uri($this->getName());
$parts = array();
$parts[] = 'file';
$parts[] = 'data';
// If this is an instanced install, add the instance identifier to the URI.
// Instanced configurations behind a CDN may not be able to control the
// request domain used by the CDN (as with AWS CloudFront). Embedding the
// instance identity in the path allows us to distinguish between requests
// originating from different instances but served through the same CDN.
$instance = PhabricatorEnv::getEnvConfig('cluster.instance');
if (strlen($instance)) {
$parts[] = '@'.$instance;
}
$parts[] = $this->getSecretKey();
$parts[] = $this->getPHID();
if ($token) {
$parts[] = $token;
}
$parts[] = $name;
$path = '/'.implode('/', $parts);
// If this file is only partially uploaded, we're just going to return a
// local URI to make sure that Ajax works, since the page is inevitably
// going to give us an error back.
if ($this->getIsPartial()) {
return PhabricatorEnv::getURI($path);
} else {
return PhabricatorEnv::getCDNURI($path);
}
}
/**
* Get the CDN URI for this file, including a one-time-use security token.
*
*/
public function getCDNURIWithToken() {
if (!$this->getPHID()) {
throw new Exception(
'You must save a file before you can generate a CDN URI.');
}
return $this->getCDNURI($this->generateOneTimeToken());
}
public function getInfoURI() {
return '/'.$this->getMonogram();
}
public function getBestURI() {
if ($this->isViewableInBrowser()) {
return $this->getViewURI();
} else {
return $this->getInfoURI();
}
}
public function getDownloadURI() {
$uri = id(new PhutilURI($this->getViewURI()))
->setQueryParam('download', true);
return (string) $uri;
}
private function getTransformedURI($transform) {
$parts = array();
$parts[] = 'file';
$parts[] = 'xform';
$instance = PhabricatorEnv::getEnvConfig('cluster.instance');
if (strlen($instance)) {
$parts[] = '@'.$instance;
}
$parts[] = $transform;
$parts[] = $this->getPHID();
$parts[] = $this->getSecretKey();
$path = implode('/', $parts);
$path = $path.'/';
return PhabricatorEnv::getCDNURI($path);
}
public function getProfileThumbURI() {
return $this->getTransformedURI('thumb-profile');
}
public function getThumb60x45URI() {
return $this->getTransformedURI('thumb-60x45');
}
public function getThumb160x120URI() {
return $this->getTransformedURI('thumb-160x120');
}
public function getPreview100URI() {
return $this->getTransformedURI('preview-100');
}
public function getPreview220URI() {
return $this->getTransformedURI('preview-220');
}
public function getThumb220x165URI() {
return $this->getTransfomredURI('thumb-220x165');
}
public function getThumb280x210URI() {
return $this->getTransformedURI('thumb-280x210');
}
public function isViewableInBrowser() {
return ($this->getViewableMimeType() !== null);
}
public function isViewableImage() {
if (!$this->isViewableInBrowser()) {
return false;
}
$mime_map = PhabricatorEnv::getEnvConfig('files.image-mime-types');
$mime_type = $this->getMimeType();
return idx($mime_map, $mime_type);
}
public function isAudio() {
if (!$this->isViewableInBrowser()) {
return false;
}
$mime_map = PhabricatorEnv::getEnvConfig('files.audio-mime-types');
$mime_type = $this->getMimeType();
return idx($mime_map, $mime_type);
}
public function isTransformableImage() {
// NOTE: The way the 'gd' extension works in PHP is that you can install it
// with support for only some file types, so it might be able to handle
// PNG but not JPEG. Try to generate thumbnails for whatever we can. Setup
// warns you if you don't have complete support.
$matches = null;
$ok = preg_match(
'@^image/(gif|png|jpe?g)@',
$this->getViewableMimeType(),
$matches);
if (!$ok) {
return false;
}
switch ($matches[1]) {
case 'jpg';
case 'jpeg':
return function_exists('imagejpeg');
break;
case 'png':
return function_exists('imagepng');
break;
case 'gif':
return function_exists('imagegif');
break;
default:
throw new Exception('Unknown type matched as image MIME type.');
}
}
public static function getTransformableImageFormats() {
$supported = array();
if (function_exists('imagejpeg')) {
$supported[] = 'jpg';
}
if (function_exists('imagepng')) {
$supported[] = 'png';
}
if (function_exists('imagegif')) {
$supported[] = 'gif';
}
return $supported;
}
public function instantiateStorageEngine() {
return self::buildEngine($this->getStorageEngine());
}
public static function buildEngine($engine_identifier) {
$engines = self::buildAllEngines();
foreach ($engines as $engine) {
if ($engine->getEngineIdentifier() == $engine_identifier) {
return $engine;
}
}
throw new Exception(
"Storage engine '{$engine_identifier}' could not be located!");
}
public static function buildAllEngines() {
$engines = id(new PhutilSymbolLoader())
->setType('class')
->setConcreteOnly(true)
->setAncestorClass('PhabricatorFileStorageEngine')
->selectAndLoadSymbols();
$results = array();
foreach ($engines as $engine_class) {
$results[] = newv($engine_class['name'], array());
}
return $results;
}
public function getViewableMimeType() {
$mime_map = PhabricatorEnv::getEnvConfig('files.viewable-mime-types');
$mime_type = $this->getMimeType();
$mime_parts = explode(';', $mime_type);
$mime_type = trim(reset($mime_parts));
return idx($mime_map, $mime_type);
}
public function getDisplayIconForMimeType() {
$mime_map = PhabricatorEnv::getEnvConfig('files.icon-mime-types');
$mime_type = $this->getMimeType();
return idx($mime_map, $mime_type, 'fa-file-o');
}
public function validateSecretKey($key) {
return ($key == $this->getSecretKey());
}
public function generateSecretKey() {
return Filesystem::readRandomCharacters(20);
}
public function updateDimensions($save = true) {
if (!$this->isViewableImage()) {
throw new Exception(
'This file is not a viewable image.');
}
if (!function_exists('imagecreatefromstring')) {
throw new Exception(
'Cannot retrieve image information.');
}
$data = $this->loadFileData();
$img = imagecreatefromstring($data);
if ($img === false) {
throw new Exception(
'Error when decoding image.');
}
$this->metadata[self::METADATA_IMAGE_WIDTH] = imagesx($img);
$this->metadata[self::METADATA_IMAGE_HEIGHT] = imagesy($img);
if ($save) {
$this->save();
}
return $this;
}
public function copyDimensions(PhabricatorFile $file) {
$metadata = $file->getMetadata();
$width = idx($metadata, self::METADATA_IMAGE_WIDTH);
if ($width) {
$this->metadata[self::METADATA_IMAGE_WIDTH] = $width;
}
$height = idx($metadata, self::METADATA_IMAGE_HEIGHT);
if ($height) {
$this->metadata[self::METADATA_IMAGE_HEIGHT] = $height;
}
return $this;
}
/**
* Load (or build) the {@class:PhabricatorFile} objects for builtin file
* resources. The builtin mechanism allows files shipped with Phabricator
* to be treated like normal files so that APIs do not need to special case
* things like default images or deleted files.
*
* Builtins are located in `resources/builtin/` and identified by their
* name.
*
* @param PhabricatorUser Viewing user.
* @param list<string> List of builtin file names.
* @return dict<string, PhabricatorFile> Dictionary of named builtins.
*/
public static function loadBuiltins(PhabricatorUser $user, array $names) {
$specs = array();
foreach ($names as $name) {
$specs[] = array(
'originalPHID' => PhabricatorPHIDConstants::PHID_VOID,
'transform' => 'builtin:'.$name,
);
}
// NOTE: Anyone is allowed to access builtin files.
$files = id(new PhabricatorFileQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withTransforms($specs)
->execute();
$files = mpull($files, null, 'getName');
$root = dirname(phutil_get_library_root('phabricator'));
$root = $root.'/resources/builtin/';
$build = array();
foreach ($names as $name) {
if (isset($files[$name])) {
continue;
}
// This is just a sanity check to prevent loading arbitrary files.
if (basename($name) != $name) {
throw new Exception("Invalid builtin name '{$name}'!");
}
$path = $root.$name;
if (!Filesystem::pathExists($path)) {
throw new Exception("Builtin '{$path}' does not exist!");
}
$data = Filesystem::readFile($path);
$params = array(
'name' => $name,
'ttl' => time() + (60 * 60 * 24 * 7),
'canCDN' => true,
'builtin' => $name,
);
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
$file = PhabricatorFile::newFromFileData($data, $params);
$xform = id(new PhabricatorTransformedFile())
->setOriginalPHID(PhabricatorPHIDConstants::PHID_VOID)
->setTransform('builtin:'.$name)
->setTransformedPHID($file->getPHID())
->save();
unset($unguarded);
$file->attachObjectPHIDs(array());
$file->attachObjects(array());
$files[$name] = $file;
}
return $files;
}
/**
* Convenience wrapper for @{method:loadBuiltins}.
*
* @param PhabricatorUser Viewing user.
* @param string Single builtin name to load.
* @return PhabricatorFile Corresponding builtin file.
*/
public static function loadBuiltin(PhabricatorUser $user, $name) {
return idx(self::loadBuiltins($user, array($name)), $name);
}
public function getObjects() {
return $this->assertAttached($this->objects);
}
public function attachObjects(array $objects) {
$this->objects = $objects;
return $this;
}
public function getObjectPHIDs() {
return $this->assertAttached($this->objectPHIDs);
}
public function attachObjectPHIDs(array $object_phids) {
$this->objectPHIDs = $object_phids;
return $this;
}
public function getOriginalFile() {
return $this->assertAttached($this->originalFile);
}
public function attachOriginalFile(PhabricatorFile $file = null) {
$this->originalFile = $file;
return $this;
}
public function getImageHeight() {
if (!$this->isViewableImage()) {
return null;
}
return idx($this->metadata, self::METADATA_IMAGE_HEIGHT);
}
public function getImageWidth() {
if (!$this->isViewableImage()) {
return null;
}
return idx($this->metadata, self::METADATA_IMAGE_WIDTH);
}
public function getCanCDN() {
if (!$this->isViewableImage()) {
return false;
}
return idx($this->metadata, self::METADATA_CAN_CDN);
}
public function setCanCDN($can_cdn) {
$this->metadata[self::METADATA_CAN_CDN] = $can_cdn ? 1 : 0;
return $this;
}
public function isBuiltin() {
return ($this->getBuiltinName() !== null);
}
public function getBuiltinName() {
return idx($this->metadata, self::METADATA_BUILTIN);
}
public function setBuiltinName($name) {
$this->metadata[self::METADATA_BUILTIN] = $name;
return $this;
}
protected function generateOneTimeToken() {
$key = Filesystem::readRandomCharacters(16);
// Save the new secret.
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
$token = id(new PhabricatorAuthTemporaryToken())
->setObjectPHID($this->getPHID())
->setTokenType(self::ONETIME_TEMPORARY_TOKEN_TYPE)
->setTokenExpires(time() + phutil_units('1 hour in seconds'))
->setTokenCode(PhabricatorHash::digest($key))
->save();
unset($unguarded);
return $key;
}
public function validateOneTimeToken($token_code) {
$token = id(new PhabricatorAuthTemporaryTokenQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withObjectPHIDs(array($this->getPHID()))
->withTokenTypes(array(self::ONETIME_TEMPORARY_TOKEN_TYPE))
->withExpired(false)
->withTokenCodes(array(PhabricatorHash::digest($token_code)))
->executeOne();
return $token;
}
/**
* Write the policy edge between this file and some object.
*
* @param phid Object PHID to attach to.
* @return this
*/
public function attachToObject($phid) {
$edge_type = PhabricatorObjectHasFileEdgeType::EDGECONST;
id(new PhabricatorEdgeEditor())
->addEdge($phid, $edge_type, $this->getPHID())
->save();
return $this;
}
/**
* Remove the policy edge between this file and some object.
*
* @param phid Object PHID to detach from.
* @return this
*/
public function detachFromObject($phid) {
$edge_type = PhabricatorObjectHasFileEdgeType::EDGECONST;
id(new PhabricatorEdgeEditor())
->removeEdge($phid, $edge_type, $this->getPHID())
->save();
return $this;
}
/**
* Configure a newly created file object according to specified parameters.
*
* This method is called both when creating a file from fresh data, and
* when creating a new file which reuses existing storage.
*
* @param map<string, wild> Bag of parameters, see @{class:PhabricatorFile}
* for documentation.
* @return this
*/
private function readPropertiesFromParameters(array $params) {
$file_name = idx($params, 'name');
$file_name = self::normalizeFileName($file_name);
$this->setName($file_name);
$author_phid = idx($params, 'authorPHID');
$this->setAuthorPHID($author_phid);
$file_ttl = idx($params, 'ttl');
$this->setTtl($file_ttl);
$view_policy = idx($params, 'viewPolicy');
if ($view_policy) {
$this->setViewPolicy($params['viewPolicy']);
}
$is_explicit = (idx($params, 'isExplicitUpload') ? 1 : 0);
$this->setIsExplicitUpload($is_explicit);
$can_cdn = idx($params, 'canCDN');
if ($can_cdn) {
$this->setCanCDN(true);
}
$builtin = idx($params, 'builtin');
if ($builtin) {
$this->setBuiltinName($builtin);
}
$mime_type = idx($params, 'mime-type');
if ($mime_type) {
$this->setMimeType($mime_type);
}
return $this;
}
public function getRedirectResponse() {
$uri = $this->getBestURI();
// TODO: This is a bit iffy. Sometimes, getBestURI() returns a CDN URI
// (if the file is a viewable image) and sometimes a local URI (if not).
// For now, just detect which one we got and configure the response
// appropriately. In the long run, if this endpoint is served from a CDN
// domain, we can't issue a local redirect to an info URI (which is not
// present on the CDN domain). We probably never actually issue local
// redirects here anyway, since we only ever transform viewable images
// right now.
$is_external = strlen(id(new PhutilURI($uri))->getDomain());
return id(new AphrontRedirectResponse())
->setIsExternal($is_external)
->setURI($uri);
}
/* -( PhabricatorApplicationTransactionInterface )------------------------- */
public function getApplicationTransactionEditor() {
return new PhabricatorFileEditor();
}
public function getApplicationTransactionObject() {
return $this;
}
public function getApplicationTransactionTemplate() {
return new PhabricatorFileTransaction();
}
public function willRenderTimeline(
PhabricatorApplicationTransactionView $timeline,
AphrontRequest $request) {
return $timeline;
}
/* -( PhabricatorPolicyInterface Implementation )-------------------------- */
public function getCapabilities() {
return array(
PhabricatorPolicyCapability::CAN_VIEW,
PhabricatorPolicyCapability::CAN_EDIT,
);
}
public function getPolicy($capability) {
switch ($capability) {
case PhabricatorPolicyCapability::CAN_VIEW:
if ($this->isBuiltin()) {
return PhabricatorPolicies::getMostOpenPolicy();
}
return $this->getViewPolicy();
case PhabricatorPolicyCapability::CAN_EDIT:
return PhabricatorPolicies::POLICY_NOONE;
}
}
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
$viewer_phid = $viewer->getPHID();
if ($viewer_phid) {
if ($this->getAuthorPHID() == $viewer_phid) {
return true;
}
}
switch ($capability) {
case PhabricatorPolicyCapability::CAN_VIEW:
// If you can see the file this file is a transform of, you can see
// this file.
if ($this->getOriginalFile()) {
return true;
}
// If you can see any object this file is attached to, you can see
// the file.
return (count($this->getObjects()) > 0);
}
return false;
}
public function describeAutomaticCapability($capability) {
$out = array();
$out[] = pht('The user who uploaded a file can always view and edit it.');
switch ($capability) {
case PhabricatorPolicyCapability::CAN_VIEW:
$out[] = pht(
'Files attached to objects are visible to users who can view '.
'those objects.');
$out[] = pht(
'Thumbnails are visible only to users who can view the original '.
'file.');
break;
}
return $out;
}
/* -( PhabricatorSubscribableInterface Implementation )-------------------- */
public function isAutomaticallySubscribed($phid) {
return ($this->authorPHID == $phid);
}
public function shouldShowSubscribersProperty() {
return true;
}
public function shouldAllowSubscription($phid) {
return true;
}
/* -( PhabricatorTokenReceiverInterface )---------------------------------- */
public function getUsersToNotifyOfTokenGiven() {
return array(
$this->getAuthorPHID(),
);
}
/* -( PhabricatorDestructibleInterface )----------------------------------- */
public function destroyObjectPermanently(
PhabricatorDestructionEngine $engine) {
$this->openTransaction();
$this->delete();
$this->saveTransaction();
}
}
diff --git a/src/docs/user/configuration/configuring_file_storage.diviner b/src/docs/user/configuration/configuring_file_storage.diviner
index 0cb517eed..9aa5b9ad4 100644
--- a/src/docs/user/configuration/configuring_file_storage.diviner
+++ b/src/docs/user/configuration/configuring_file_storage.diviner
@@ -1,104 +1,198 @@
@title Configuring File Storage
@group config
-Setup how Phabricator will store files.
+Setup file storage and support for large files.
Overview
========
-Phabricator allows users to upload files, and several applications use file
-storage (for instance, Maniphest allows you to attach files to tasks). You can
-configure several different storage systems.
+This document describes how to configure Phabricator to support large file
+uploads, and how to choose where Phabricator stores files.
-| System | Setup | Cost | Notes |
+There are two major things to configure:
+
+ - set up PHP and your HTTP server to accept large requests;
+ - choose and configure a storage engine.
+
+The following sections will guide you through this configuration.
+
+
+How Phabricator Stores Files
+============================
+
+Phabricator stores files in "storage engines", which are modular backends
+that implement access to some storage system (like MySQL, the filesystem, or
+a cloud storage service like Amazon S3).
+
+Phabricator stores large files by breaking them up into many chunks (a few
+megabytes in size) and storing the chunks in an underlying storage engine.
+This makes it easier to implement new storage engines and gives Phabricator
+more flexibility in managing file data.
+
+The first section of this document discusses configuring your install so that
+PHP and your HTTP server will accept requests which are larger than the size of
+one file chunk. Without this configuration, file chunk data will be rejected.
+
+The second section discusses choosing and configuring storage engines, so data
+is stored where you want it to be.
+
+
+Configuring Upload Limits
+=========================
+
+File uploads are limited by several pieces of configuration at different layers
+of the stack. Generally, the minimum value of all the limits is the effective
+one.
+
+To upload large files, you need to increase all the limits to at least
+**32MB**. This will allow you to upload file chunks, which will let Phabricator
+store arbitrarily large files.
+
+The settings which limit file uploads are:
+
+**HTTP Server**: The HTTP server may set a limit on the maximum request size.
+If you exceed this limit, you'll see a default server page with an HTTP error.
+These directives limit the total size of the request body, so they must be
+somewhat larger than the desired maximum filesize.
+
+ - **Apache**: Apache limits requests with the Apache `LimitRequestBody`
+ directive.
+ - **nginx**: nginx limits requests with the nginx `client_max_body_size`
+ directive. This often defaults to `1M`.
+ - **lighttpd**: lighttpd limits requests with the lighttpd
+ `server.max-request-size` directive.
+
+Set the applicable limit to at least **32MB**. Phabricator can not read these
+settings, so it can not raise setup warnings if they are misconfigured.
+
+**PHP**: PHP has several directives which limit uploads. These directives are
+found in `php.ini`.
+
+ - **post_max_size**: Maximum POST request size PHP will accept. If you
+ exceed this, Phabricator will give you a useful error. This often defaults
+ to `8M`. Set this to at least `32MB`. Phabricator will give you a setup
+ warning about this if it is set too low.
+ - **memory_limit**: For some uploads, file data will be read into memory
+ before Phabricator can adjust the memory limit. If you exceed this, PHP
+ may give you a useful error, depending on your configuration. It is
+ recommended that you set this to `-1` to disable it. Phabricator will
+ give you a setup warning about this if it is set too low.
+
+You may also want to configure these PHP options:
+
+ - **max_input_vars**: When files are uploaded via HTML5 drag and drop file
+ upload APIs, PHP parses the file body as though it contained normal POST
+ parameters, and may trigger `max_input_vars` if a file has a lot of
+ brackets in it. You may need to set it to some astronomically high value.
+ - **upload_max_filesize**: Maximum file size PHP will accept in a raw file
+ upload. This is not normally used when uploading files via drag-and-drop,
+ but affects some other kinds of file uploads. If you exceed this,
+ Phabricator will give you a useful error. This often defaults to `2M`. Set
+ this to at least `32MB`.
+
+Once you've adjusted all this configuration, your server will be able to
+receive chunk uploads. As long as you have somewhere to store them, this will
+enable you to store arbitrarily large files.
+
+
+Storage Engines
+===============
+
+Phabricator supports several different file storage engines:
+
+| Engine | Setup | Cost | Notes |
|========|=======|======|=======|
| MySQL | Automatic | Free | May not scale well. |
| Local Disk | Easy | Free | Does not scale well. |
| Amazon S3 | Easy | Cheap | Scales well. |
| Custom | Hard | Varies | Implement a custom storage engine. |
+You can review available storage engines and their configuration by navigating
+to {nav Applications > Files > Help/Options > Storage Engines} in the web UI.
+
By default, Phabricator is configured to store files up to 1MB in MySQL, and
reject files larger than 1MB. To store larger files, you can either:
- - configure local disk storage; or
- - configure Amazon S3 storage; or
- - raise the limits on MySQL.
+ - increase the MySQL limit to at least 8MB; or
+ - configure another storage engine.
-See the rest of this document for some additional discussion of engines.
+Doing either of these will enable the chunk storage engine and support for
+arbitrarily large files.
-You don't have to fully configure this immediately, the defaults are okay until
-you need to upload larger files and it's relatively easy to port files between
-storage engines later.
-
-Storage Engines
-===============
+The remaining sections of this document discuss the available storage engines
+and how to configure them.
-Builtin storage engines and information on how to configure them.
-== MySQL ==
+Engine: MySQL
+=============
- - **Pros**: Fast, no setup required.
- - **Cons**: Storing files in a database is a classic bad idea. Does not scale
- well. Maximum file size is limited.
+ - **Pros**: Low latency, no setup required.
+ - **Cons**: Storing files in a database is a classic bad idea. May become
+ difficult to administrate if you have a large amount of data.
MySQL storage is configured by default, for files up to (just under) 1MB. You
can configure it with these keys:
- `storage.mysql-engine.max-size`: Change the filesize limit. Set to 0
to disable.
For most installs, it is reasonable to leave this engine as-is and let small
files (like thumbnails and profile images) be stored in MySQL, which is usually
-the lowest-latency filestore.
+the lowest-latency filestore, even if you configure another storage engine.
-To support larger files, configure another engine or increase this limit.
+To support large files, increase this limit to at least **8MB**. This will
+activate chunk storage in MySQL.
-== Local Disk ==
+Engine: Local Disk
+==================
- - **Pros**: Very simple. Almost no setup required.
+ - **Pros**: Simple to setup.
- **Cons**: Doesn't scale to multiple web frontends without NFS.
-To upload larger files:
+To configure file storage on the local disk, set:
- `storage.local-disk.path`: Set to some writable directory on local disk.
Make that directory.
-== Amazon S3 ==
+Engine: Amazon S3
+=================
- **Pros**: Scales well.
- - **Cons**: More complicated and expensive than other approaches.
+ - **Cons**: Slightly more complicated than other engines, not free.
-To enable file storage in S3, set these key:
+To enable file storage in S3, set these keys:
- - ##amazon-s3.access-key## Your AWS access key.
- - ##amazon-s3.secret-key## Your AWS secret key.
- - ##storage.s3.bucket## S3 bucket name where files should be stored.
+ - `amazon-s3.access-key`: Your AWS access key.
+ - `amazon-s3.secret-key`: Your AWS secret key.
+ - `storage.s3.bucket`: S3 bucket name where files should be stored.
-= Testing Storage Engines =
+Testing Storage Engines
+=======================
-You can test that things are correctly configured by going to the Files
-application (##/file/##) and uploading files.
+You can test that things are correctly configured by dragging and dropping
+a file onto the Phabricator home page. If engines have been configured
+properly, the file should upload.
-= Migrating Files Between Engines =
+Migrating Files Between Engines
+===============================
If you want to move files between storage engines, you can use the `bin/files`
script to perform migrations. For example, suppose you previously used MySQL but
recently set up S3 and want to migrate all your files there. First, migrate one
file to make sure things work:
phabricator/ $ ./bin/files migrate --engine amazon-s3 F12345
If that works properly, you can then migrate everything:
phabricator/ $ ./bin/files migrate --engine amazon-s3 --all
You can use `--dry-run` to show which migrations would be performed without
taking any action. Run `bin/files help` for more options and information.
-= Next Steps =
+Next Steps
+==========
Continue by:
- - configuring file size upload limits with
- @{article:Configuring File Upload Limits}; or
- returning to the @{article:Configuration Guide}.
diff --git a/src/docs/user/configuration/configuring_file_upload_limits.diviner b/src/docs/user/configuration/configuring_file_upload_limits.diviner
deleted file mode 100644
index 1bfbc6104..000000000
--- a/src/docs/user/configuration/configuring_file_upload_limits.diviner
+++ /dev/null
@@ -1,77 +0,0 @@
-@title Configuring File Upload Limits
-@group config
-
-Explains limits on file upload sizes.
-
-= Overview =
-
-File uploads are limited by a large number of pieces of configuration, at
-multiple layers of the application. Generally, the minimum value of all the
-limits is the effective one. To upload large files, you need to increase all
-the limits above the maximum file size you want to support. The settings which
-limit uploads are:
-
- - **HTTP Server**: The HTTP server may set a limit on the maximum request
- size. If you exceed this limit, you'll see a default server page with an
- HTTP error. These directives limit the total size of the request body,
- so they must be somewhat larger than the desired maximum filesize.
- - **Apache**: Apache limits requests with the Apache `LimitRequestBody`
- directive.
- - **nginx**: nginx limits requests with the nginx `client_max_body_size`
- directive. This often defaults to `1M`.
- - **lighttpd**: lighttpd limits requests with the lighttpd
- `server.max-request-size` directive.
- - **PHP**: PHP has several directives which limit uploads. These directives
- are found in `php.ini`.
- - **upload_max_filesize**: Maximum file size PHP will accept in a file
- upload. If you exceed this, Phabricator will give you a useful error. This
- often defaults to `2M`.
- - **post_max_size**: Maximum POST request size PHP will accept. If you
- exceed this, Phabricator will give you a useful error. This often defaults
- to `8M`.
- - **memory_limit**: For some uploads, file data will be read into memory
- before Phabricator can adjust the memory limit. If you exceed this, PHP
- may give you a useful error, depending on your configuration.
- - **max_input_vars**: When files are uploaded via HTML5 drag and drop file
- upload APIs, PHP parses the file body as though it contained normal POST
- parameters, and may trigger `max_input_vars` if a file has a lot of
- brackets in it. You may need to set it to some astronomically high value.
- - **Storage Engines**: Some storage engines can be configured not to accept
- files over a certain size. To upload a file, you must have at least one
- configured storage engine which can accept it. Phabricator should give you
- useful errors if any of these fail.
- - **MySQL Engine**: Upload size is limited by the Phabricator setting
- `storage.mysql-engine.max-size`.
- - **Amazon S3**: Upload size is limited by Phabricator's implementation to
- `5G`.
- - **Local Disk**: Upload size is limited only by free disk space.
- - **Resource Constraints**: File uploads are limited by resource constraints
- on the application server. In particular, some uploaded files are written
- to disk in their entirety before being moved to storage engines, and all
- uploaded files are read into memory before being moved. These hard limits
- should be large for most servers, but will fundamentally prevent Phabricator
- from processing truly enormous files (GB/TB scale). Phabricator is probably
- not the best application for this in any case.
- - **Phabricator Master Limit**: The master limit, `storage.upload-size-limit`,
- is used to show upload limits in the UI.
-
-Phabricator can't read some of these settings, so it can't figure out what the
-current limit is or be much help at all in configuring it. Thus, you need to
-manually configure all of these limits and then tell Phabricator what you set
-them to. Follow these steps:
-
- - Pick some limit you want to set, like `100M`.
- - Configure all of the settings mentioned above to be a bit bigger than the
- limit you want to enforce (**note that there are some security implications
- to raising these limits**; principally, your server may become easier to
- attack with a denial-of-service).
- - Set `storage.upload-size-limit` to the limit you want.
- - The UI should now show your limit.
- - Upload a big file to make sure it works.
-
-= Next Steps =
-
-Continue by:
-
- - configuring file storage with @{article:Configuring File Storage}; or
- - returning to the @{article:Configuration Guide}.